diff options
author | robot-piglet <robot-piglet@yandex-team.com> | 2023-10-31 10:22:52 +0300 |
---|---|---|
committer | robot-piglet <robot-piglet@yandex-team.com> | 2023-10-31 10:39:15 +0300 |
commit | f87b58a8def78eb3eb7d46bee3e0b16a5293f8c4 (patch) | |
tree | 366ec2807aad9e0a3368eedb2a436f61b0eb7aa6 | |
parent | cf679e7e14de7680929174ff5647c96b739c31f7 (diff) | |
download | ydb-f87b58a8def78eb3eb7d46bee3e0b16a5293f8c4.tar.gz |
Intermediate changes
64 files changed, 1836 insertions, 1389 deletions
diff --git a/contrib/python/hypothesis/py3/.dist-info/METADATA b/contrib/python/hypothesis/py3/.dist-info/METADATA index 1eb27e8c70..fea15f2238 100644 --- a/contrib/python/hypothesis/py3/.dist-info/METADATA +++ b/contrib/python/hypothesis/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: hypothesis -Version: 6.78.1 +Version: 6.88.1 Summary: A library for property-based testing Home-page: https://hypothesis.works Author: David R. MacIver and Zac Hatfield-Dodds @@ -22,7 +22,6 @@ Classifier: Operating System :: Microsoft :: Windows Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.7 Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 @@ -32,58 +31,57 @@ Classifier: Programming Language :: Python :: Implementation :: PyPy Classifier: Topic :: Education :: Testing Classifier: Topic :: Software Development :: Testing Classifier: Typing :: Typed -Requires-Python: >=3.7 +Requires-Python: >=3.8 Description-Content-Type: text/x-rst License-File: LICENSE.txt -Requires-Dist: attrs (>=19.2.0) -Requires-Dist: sortedcontainers (<3.0.0,>=2.1.0) -Requires-Dist: exceptiongroup (>=1.0.0) ; python_version < "3.11" +Requires-Dist: attrs >=19.2.0 +Requires-Dist: sortedcontainers <3.0.0,>=2.1.0 +Requires-Dist: exceptiongroup >=1.0.0 ; python_version < "3.11" Provides-Extra: all -Requires-Dist: black (>=19.10b0) ; extra == 'all' -Requires-Dist: click (>=7.0) ; extra == 'all' -Requires-Dist: django (>=3.2) ; extra == 'all' -Requires-Dist: dpcontracts (>=0.4) ; extra == 'all' -Requires-Dist: lark (>=0.10.1) ; extra == 'all' -Requires-Dist: libcst (>=0.3.16) ; extra == 'all' -Requires-Dist: numpy (>=1.16.0) ; extra == 'all' -Requires-Dist: pandas (>=1.1) ; extra == 'all' -Requires-Dist: pytest (>=4.6) ; extra == 'all' -Requires-Dist: python-dateutil (>=1.4) ; extra == 'all' -Requires-Dist: pytz (>=2014.1) ; extra == 'all' -Requires-Dist: redis (>=3.0.0) ; extra == 'all' -Requires-Dist: rich (>=9.0.0) ; extra == 'all' -Requires-Dist: importlib-metadata (>=3.6) ; (python_version < "3.8") and extra == 'all' -Requires-Dist: backports.zoneinfo (>=0.2.1) ; (python_version < "3.9") and extra == 'all' -Requires-Dist: tzdata (>=2023.3) ; (sys_platform == "win32") and extra == 'all' +Requires-Dist: black >=19.10b0 ; extra == 'all' +Requires-Dist: click >=7.0 ; extra == 'all' +Requires-Dist: django >=3.2 ; extra == 'all' +Requires-Dist: dpcontracts >=0.4 ; extra == 'all' +Requires-Dist: lark >=0.10.1 ; extra == 'all' +Requires-Dist: libcst >=0.3.16 ; extra == 'all' +Requires-Dist: numpy >=1.17.3 ; extra == 'all' +Requires-Dist: pandas >=1.1 ; extra == 'all' +Requires-Dist: pytest >=4.6 ; extra == 'all' +Requires-Dist: python-dateutil >=1.4 ; extra == 'all' +Requires-Dist: pytz >=2014.1 ; extra == 'all' +Requires-Dist: redis >=3.0.0 ; extra == 'all' +Requires-Dist: rich >=9.0.0 ; extra == 'all' +Requires-Dist: backports.zoneinfo >=0.2.1 ; (python_version < "3.9") and extra == 'all' +Requires-Dist: tzdata >=2023.3 ; (sys_platform == "win32") and extra == 'all' Provides-Extra: cli -Requires-Dist: click (>=7.0) ; extra == 'cli' -Requires-Dist: black (>=19.10b0) ; extra == 'cli' -Requires-Dist: rich (>=9.0.0) ; extra == 'cli' +Requires-Dist: click >=7.0 ; extra == 'cli' +Requires-Dist: black >=19.10b0 ; extra == 'cli' +Requires-Dist: rich >=9.0.0 ; extra == 'cli' Provides-Extra: codemods -Requires-Dist: libcst (>=0.3.16) ; extra == 'codemods' +Requires-Dist: libcst >=0.3.16 ; extra == 'codemods' Provides-Extra: dateutil -Requires-Dist: python-dateutil (>=1.4) ; extra == 'dateutil' +Requires-Dist: python-dateutil >=1.4 ; extra == 'dateutil' Provides-Extra: django -Requires-Dist: django (>=3.2) ; extra == 'django' +Requires-Dist: django >=3.2 ; extra == 'django' Provides-Extra: dpcontracts -Requires-Dist: dpcontracts (>=0.4) ; extra == 'dpcontracts' +Requires-Dist: dpcontracts >=0.4 ; extra == 'dpcontracts' Provides-Extra: ghostwriter -Requires-Dist: black (>=19.10b0) ; extra == 'ghostwriter' +Requires-Dist: black >=19.10b0 ; extra == 'ghostwriter' Provides-Extra: lark -Requires-Dist: lark (>=0.10.1) ; extra == 'lark' +Requires-Dist: lark >=0.10.1 ; extra == 'lark' Provides-Extra: numpy -Requires-Dist: numpy (>=1.16.0) ; extra == 'numpy' +Requires-Dist: numpy >=1.17.3 ; extra == 'numpy' Provides-Extra: pandas -Requires-Dist: pandas (>=1.1) ; extra == 'pandas' +Requires-Dist: pandas >=1.1 ; extra == 'pandas' Provides-Extra: pytest -Requires-Dist: pytest (>=4.6) ; extra == 'pytest' +Requires-Dist: pytest >=4.6 ; extra == 'pytest' Provides-Extra: pytz -Requires-Dist: pytz (>=2014.1) ; extra == 'pytz' +Requires-Dist: pytz >=2014.1 ; extra == 'pytz' Provides-Extra: redis -Requires-Dist: redis (>=3.0.0) ; extra == 'redis' +Requires-Dist: redis >=3.0.0 ; extra == 'redis' Provides-Extra: zoneinfo -Requires-Dist: backports.zoneinfo (>=0.2.1) ; (python_version < "3.9") and extra == 'zoneinfo' -Requires-Dist: tzdata (>=2023.3) ; (sys_platform == "win32") and extra == 'zoneinfo' +Requires-Dist: backports.zoneinfo >=0.2.1 ; (python_version < "3.9") and extra == 'zoneinfo' +Requires-Dist: tzdata >=2023.3 ; (sys_platform == "win32") and extra == 'zoneinfo' ========== Hypothesis diff --git a/contrib/python/hypothesis/py3/_hypothesis_pytestplugin.py b/contrib/python/hypothesis/py3/_hypothesis_pytestplugin.py index 765eee3aac..3bb2535f3b 100644 --- a/contrib/python/hypothesis/py3/_hypothesis_pytestplugin.py +++ b/contrib/python/hypothesis/py3/_hypothesis_pytestplugin.py @@ -91,7 +91,7 @@ if tuple(map(int, pytest.__version__.split(".")[:2])) < (4, 6): # pragma: no co Note that the pytest developers no longer support your version either! Disabling the Hypothesis pytest plugin... """ - warnings.warn(PYTEST_TOO_OLD_MESSAGE % (pytest.__version__,)) + warnings.warn(PYTEST_TOO_OLD_MESSAGE % (pytest.__version__,), stacklevel=1) else: @@ -168,7 +168,7 @@ else: and Phase.explain not in settings.default.phases ): name = f"{settings._current_profile}-with-explain-phase" - phases = settings.default.phases + (Phase.explain,) + phases = (*settings.default.phases, Phase.explain) settings.register_profile(name, phases=phases) settings.load_profile(name) @@ -191,9 +191,9 @@ else: from hypothesis.internal.detection import is_hypothesis_test # See https://github.com/pytest-dev/pytest/issues/9159 - # TODO: add `pytest_version >= (7, 2) or` once the issue above is fixed. core.pytest_shows_exceptiongroups = ( - item.config.getoption("tbstyle", "auto") == "native" + getattr(pytest, "version_tuple", ())[:2] >= (7, 2) + or item.config.getoption("tbstyle", "auto") == "native" ) core.running_under_pytest = True @@ -221,12 +221,12 @@ else: ("reproduce_example", "_hypothesis_internal_use_reproduce_failure"), ]: if hasattr(item.obj, attribute): - from hypothesis.errors import InvalidArgument # noqa: F811 + from hypothesis.errors import InvalidArgument raise_hypothesis_usage_error(message % (name,)) yield else: - from hypothesis import HealthCheck, settings + from hypothesis import HealthCheck, settings as Settings from hypothesis.internal.escalation import current_pytest_item from hypothesis.internal.healthcheck import fail_health_check from hypothesis.reporting import with_reporter @@ -237,16 +237,15 @@ else: # work, the test object is probably something weird # (e.g a stateful test wrapper), so we skip the function-scoped # fixture check. - settings = getattr( # noqa: F811 - item.obj, "_hypothesis_internal_use_settings", None + settings = getattr( + item.obj, "_hypothesis_internal_use_settings", Settings.default ) # Check for suspicious use of function-scoped fixtures, but only # if the corresponding health check is not suppressed. - if ( - settings is not None - and HealthCheck.function_scoped_fixture - not in settings.suppress_health_check + fixture_params = False + if not set(settings.suppress_health_check).issuperset( + {HealthCheck.function_scoped_fixture, HealthCheck.differing_executors} ): # Warn about function-scoped fixtures, excluding autouse fixtures because # the advice is probably not actionable and the status quo seems OK... @@ -258,6 +257,7 @@ else: if argnames is None: argnames = frozenset(signature(item.function).parameters) for fx in fx_defs: + fixture_params |= bool(fx.params) if fx.argname in argnames: active_fx = item._request._get_active_fixturedef(fx.argname) if active_fx.scope == "function": @@ -267,7 +267,18 @@ else: HealthCheck.function_scoped_fixture, ) - if item.get_closest_marker("parametrize") is not None: + if fixture_params or (item.get_closest_marker("parametrize") is not None): + # Disable the differing_executors health check due to false alarms: + # see https://github.com/HypothesisWorks/hypothesis/issues/3733 + from hypothesis import settings as Settings + + fn = getattr(item.obj, "__func__", item.obj) + fn._hypothesis_internal_use_settings = Settings( + parent=settings, + suppress_health_check={HealthCheck.differing_executors} + | set(settings.suppress_health_check), + ) + # Give every parametrized test invocation a unique database key key = item.nodeid.encode() item.obj.hypothesis.inner_test._hypothesis_internal_add_digest = key @@ -329,8 +340,9 @@ else: # If there's an HTML report, include our summary stats for each test pytest_html = item.config.pluginmanager.getplugin("html") if pytest_html is not None: # pragma: no cover - report.extra = getattr(report, "extra", []) + [ - pytest_html.extras.text(stats, name="Hypothesis stats") + report.extra = [ + *getattr(report, "extra", []), + pytest_html.extras.text(stats, name="Hypothesis stats"), ] # This doesn't intrinsically have anything to do with the terminalreporter; diff --git a/contrib/python/hypothesis/py3/hypothesis/_error_if_old.py b/contrib/python/hypothesis/py3/hypothesis/_error_if_old.py index d82c32eb3e..7f0850892d 100644 --- a/contrib/python/hypothesis/py3/hypothesis/_error_if_old.py +++ b/contrib/python/hypothesis/py3/hypothesis/_error_if_old.py @@ -13,11 +13,11 @@ import sys from hypothesis.version import __version__ message = """ -Hypothesis {} requires Python 3.7 or later. +Hypothesis {} requires Python 3.8 or later. This can only happen if your packaging toolchain is older than python_requires. See https://packaging.python.org/guides/distributing-packages-using-setuptools/ """ -if sys.version_info[:2] < (3, 7): +if sys.version_info[:2] < (3, 8): raise Exception(message.format(__version__)) diff --git a/contrib/python/hypothesis/py3/hypothesis/_settings.py b/contrib/python/hypothesis/py3/hypothesis/_settings.py index bf5640545e..4751bb8086 100644 --- a/contrib/python/hypothesis/py3/hypothesis/_settings.py +++ b/contrib/python/hypothesis/py3/hypothesis/_settings.py @@ -20,7 +20,17 @@ import inspect import os import warnings from enum import Enum, EnumMeta, IntEnum, unique -from typing import TYPE_CHECKING, Any, Collection, Dict, List, Optional, TypeVar, Union +from typing import ( + TYPE_CHECKING, + Any, + ClassVar, + Collection, + Dict, + List, + Optional, + TypeVar, + Union, +) import attr @@ -128,7 +138,7 @@ class settings(metaclass=settingsMeta): """ __definitions_are_locked = False - _profiles: Dict[str, "settings"] = {} + _profiles: ClassVar[Dict[str, "settings"]] = {} __module__ = "hypothesis" def __getattr__(self, name): @@ -162,7 +172,7 @@ class settings(metaclass=settingsMeta): if database not in (not_set, None): # type: ignore raise InvalidArgument( "derandomize=True implies database=None, so passing " - f"database={database!r} too is invalid." + f"{database=} too is invalid." ) database = None @@ -191,7 +201,7 @@ class settings(metaclass=settingsMeta): if not callable(_test): raise InvalidArgument( "settings objects can be called as a decorator with @given, " - f"but decorated test={test!r} is not callable." + f"but decorated {test=} is not callable." ) if inspect.isclass(test): from hypothesis.stateful import RuleBasedStateMachine @@ -231,6 +241,7 @@ class settings(metaclass=settingsMeta): cls, name, description, + *, default, options=None, validator=None, @@ -468,6 +479,7 @@ class HealthCheck(Enum, metaclass=HealthCheckMeta): "`Healthcheck.all()` is deprecated; use `list(HealthCheck)` instead.", since="2023-04-16", has_codemod=True, + stacklevel=1, ) return list(HealthCheck) @@ -518,6 +530,18 @@ class HealthCheck(Enum, metaclass=HealthCheckMeta): This check requires the :ref:`Hypothesis pytest plugin<pytest-plugin>`, which is enabled by default when running Hypothesis inside pytest.""" + differing_executors = 10 + """Checks if :func:`@given <hypothesis.given>` has been applied to a test + which is executed by different :ref:`executors<custom-function-execution>`. + If your test function is defined as a method on a class, that class will be + your executor, and subclasses executing an inherited test is a common way + for things to go wrong. + + The correct fix is often to bring the executor instance under the control + of hypothesis by explicit parametrization over, or sampling from, + subclasses, or to refactor so that :func:`@given <hypothesis.given>` is + specified on leaf subclasses.""" + @unique class Verbosity(IntEnum): @@ -548,9 +572,7 @@ def _validate_phases(phases): settings._define_setting( "phases", - # We leave the `explain` phase disabled by default, for speed and brevity - # TODO: consider default-enabling this in CI? - default=_validate_phases(set(Phase) - {Phase.explain}), + default=tuple(Phase), description=( "Control which phases should be run. " "See :ref:`the full documentation for more details <phases>`" @@ -601,6 +623,7 @@ def validate_health_check_suppressions(suppressions): f"The {s.name} health check is deprecated, because this is always an error.", since="2023-03-15", has_codemod=False, + stacklevel=2, ) return suppressions @@ -687,16 +710,18 @@ The default is ``True`` if the ``CI`` or ``TF_BUILD`` env vars are set, ``False` settings.lock_further_definitions() -def note_deprecation(message: str, *, since: str, has_codemod: bool) -> None: +def note_deprecation( + message: str, *, since: str, has_codemod: bool, stacklevel: int = 0 +) -> None: if since != "RELEASEDAY": - date = datetime.datetime.strptime(since, "%Y-%m-%d").date() + date = datetime.date.fromisoformat(since) assert datetime.date(2021, 1, 1) <= date if has_codemod: message += ( "\n The `hypothesis codemod` command-line tool can automatically " "refactor your code to fix this warning." ) - warnings.warn(HypothesisDeprecationWarning(message), stacklevel=2) + warnings.warn(HypothesisDeprecationWarning(message), stacklevel=2 + stacklevel) settings.register_profile("default", settings()) diff --git a/contrib/python/hypothesis/py3/hypothesis/configuration.py b/contrib/python/hypothesis/py3/hypothesis/configuration.py index 8f20aec7ea..6e6ab29516 100644 --- a/contrib/python/hypothesis/py3/hypothesis/configuration.py +++ b/contrib/python/hypothesis/py3/hypothesis/configuration.py @@ -9,28 +9,23 @@ # obtain one at https://mozilla.org/MPL/2.0/. import os +from pathlib import Path -__hypothesis_home_directory_default = os.path.join(os.getcwd(), ".hypothesis") +__hypothesis_home_directory_default = Path.cwd() / ".hypothesis" __hypothesis_home_directory = None def set_hypothesis_home_dir(directory): global __hypothesis_home_directory - __hypothesis_home_directory = directory - - -def mkdir_p(path): - try: - os.makedirs(path) - except OSError: - pass + __hypothesis_home_directory = None if directory is None else Path(directory) def storage_directory(*names): global __hypothesis_home_directory if not __hypothesis_home_directory: - __hypothesis_home_directory = os.getenv("HYPOTHESIS_STORAGE_DIRECTORY") + if where := os.getenv("HYPOTHESIS_STORAGE_DIRECTORY"): + __hypothesis_home_directory = Path(where) if not __hypothesis_home_directory: __hypothesis_home_directory = __hypothesis_home_directory_default - return os.path.join(__hypothesis_home_directory, *names) + return __hypothesis_home_directory.joinpath(*names) diff --git a/contrib/python/hypothesis/py3/hypothesis/control.py b/contrib/python/hypothesis/py3/hypothesis/control.py index 2229e3db0f..eb1768c565 100644 --- a/contrib/python/hypothesis/py3/hypothesis/control.py +++ b/contrib/python/hypothesis/py3/hypothesis/control.py @@ -13,6 +13,7 @@ from collections import defaultdict from typing import NoReturn, Union from hypothesis import Verbosity, settings +from hypothesis._settings import note_deprecation from hypothesis.errors import InvalidArgument, UnsatisfiedAssumption from hypothesis.internal.compat import BaseExceptionGroup from hypothesis.internal.conjecture.data import ConjectureData @@ -24,7 +25,13 @@ from hypothesis.vendor.pretty import IDKey def reject() -> NoReturn: - raise UnsatisfiedAssumption() + if _current_build_context.value is None: + note_deprecation( + "Using `reject` outside a property-based test is deprecated", + since="2023-09-25", + has_codemod=False, + ) + raise UnsatisfiedAssumption def assume(condition: object) -> bool: @@ -34,8 +41,14 @@ def assume(condition: object) -> bool: This allows you to specify properties that you *assume* will be true, and let Hypothesis try to avoid similar examples in future. """ + if _current_build_context.value is None: + note_deprecation( + "Using `assume` outside a property-based test is deprecated", + since="2023-09-25", + has_codemod=False, + ) if not condition: - raise UnsatisfiedAssumption() + raise UnsatisfiedAssumption return True @@ -62,7 +75,7 @@ def current_build_context() -> "BuildContext": class BuildContext: - def __init__(self, data, is_final=False, close_on_capture=True): + def __init__(self, data, *, is_final=False, close_on_capture=True): assert isinstance(data, ConjectureData) self.data = data self.tasks = [] @@ -206,7 +219,7 @@ def target(observation: Union[int, float], *, label: str = "") -> Union[int, flo """ check_type((int, float), observation, "observation") if not math.isfinite(observation): - raise InvalidArgument(f"observation={observation!r} must be a finite float.") + raise InvalidArgument(f"{observation=} must be a finite float.") check_type(str, label, "label") context = _current_build_context.value @@ -215,12 +228,12 @@ def target(observation: Union[int, float], *, label: str = "") -> Union[int, flo "Calling target() outside of a test is invalid. " "Consider guarding this call with `if currently_in_test_context(): ...`" ) - verbose_report(f"Saw target(observation={observation!r}, label={label!r})") + verbose_report(f"Saw target({observation!r}, {label=})") if label in context.data.target_observations: raise InvalidArgument( - f"Calling target({observation!r}, label={label!r}) would overwrite " - f"target({context.data.target_observations[label]!r}, label={label!r})" + f"Calling target({observation!r}, {label=}) would overwrite " + f"target({context.data.target_observations[label]!r}, {label=})" ) else: context.data.target_observations[label] = observation diff --git a/contrib/python/hypothesis/py3/hypothesis/core.py b/contrib/python/hypothesis/py3/hypothesis/core.py index 07cf9f897b..72d73aadb7 100644 --- a/contrib/python/hypothesis/py3/hypothesis/core.py +++ b/contrib/python/hypothesis/py3/hypothesis/core.py @@ -66,16 +66,17 @@ from hypothesis.errors import ( Unsatisfiable, UnsatisfiedAssumption, ) -from hypothesis.executors import default_new_style_executor, new_style_executor from hypothesis.internal.compat import ( PYPY, BaseExceptionGroup, + add_note, bad_django_TestCase, get_type_hints, int_from_bytes, ) from hypothesis.internal.conjecture.data import ConjectureData, Status from hypothesis.internal.conjecture.engine import BUFFER_SIZE, ConjectureRunner +from hypothesis.internal.conjecture.junkdrawer import ensure_free_stackframes from hypothesis.internal.conjecture.shrinker import sort_key from hypothesis.internal.entropy import deterministic_PRNG from hypothesis.internal.escalation import ( @@ -164,14 +165,15 @@ class example: def xfail( self, - condition: bool = True, + condition: bool = True, # noqa: FBT002 *, reason: str = "", raises: Union[ Type[BaseException], Tuple[Type[BaseException], ...] ] = BaseException, ) -> "example": - """Mark this example as an expected failure, like pytest.mark.xfail(). + """Mark this example as an expected failure, similarly to + :obj:`pytest.mark.xfail(strict=True) <pytest.mark.xfail>`. Expected-failing examples allow you to check that your test does fail on some examples, and therefore build confidence that *passing* tests are @@ -186,6 +188,37 @@ class example: @example(...).xfail(condition=sys.platform != "linux", raises=OSError) def test(x): pass + + .. note:: + + Expected-failing examples are handled separately from those generated + by strategies, so you should usually ensure that there is no overlap. + + .. code-block:: python + + @example(x=1, y=0).xfail(raises=ZeroDivisionError) + @given(x=st.just(1), y=st.integers()) # Missing `.filter(bool)`! + def test_fraction(x, y): + # This test will try the explicit example and see it fail as + # expected, then go on to generate more examples from the + # strategy. If we happen to generate y=0, the test will fail + # because only the explicit example is treated as xfailing. + x / y + + Note that this "method chaining" syntax requires Python 3.9 or later, for + :pep:`614` relaxing grammar restrictions on decorators. If you need to + support older versions of Python, you can use an identity function: + + .. code-block:: python + + def identity(x): + return x + + + @identity(example(...).xfail()) + def test(x): + pass + """ check_type(bool, condition, "condition") check_type(str, reason, "reason") @@ -199,7 +232,7 @@ class example: ) ): raise InvalidArgument( - f"raises={raises!r} must be an exception type or tuple of exception types" + f"{raises=} must be an exception type or tuple of exception types" ) if condition: self._this_example = attr.evolve( @@ -207,7 +240,7 @@ class example: ) return self - def via(self, *whence: str) -> "example": + def via(self, whence: str, /) -> "example": """Attach a machine-readable label noting whence this example came. The idea is that tools will be able to add ``@example()`` cases for you, e.g. @@ -242,26 +275,11 @@ class example: pass """ - if len(whence) != 1 or not isinstance(whence[0], str): + if not isinstance(whence, str): raise InvalidArgument(".via() must be passed a string") # This is deliberately a no-op at runtime; the tools operate on source code. return self - if sys.version_info[:2] >= (3, 8): - # We want a positional-only argument, and on Python 3.8+ we'll get it. - __sig = get_signature(via) - via = define_function_signature( - name=via.__name__, - docstring=via.__doc__, - signature=__sig.replace( - parameters=[ - p.replace(kind=inspect.Parameter.POSITIONAL_ONLY) - for p in __sig.parameters.values() - ] - ), - )(via) - del __sig - def seed(seed: Hashable) -> Callable[[TestFunc], TestFunc]: """seed: Start the test execution from a specific seed. @@ -482,11 +500,17 @@ def execute_explicit_examples(state, wrapped_test, arguments, kwargs, original_s execute_example() else: # @example(...).xfail(...) + try: execute_example() except failure_exceptions_to_catch() as err: if not isinstance(err, example.raises): raise + # Save a string form of this example; we'll warn if it's + # ever generated by the strategy (which can't be xfailed) + state.xfail_example_reprs.add( + repr_call(state.test, arguments, example_kwargs) + ) except example.raises as err: # We'd usually check this as early as possible, but it's # possible for failure_exceptions_to_catch() to grow when @@ -610,17 +634,16 @@ def process_arguments_to_given(wrapped_test, arguments, kwargs, given_kwargs, pa if is_mock(selfy): selfy = None - test_runner = new_style_executor(selfy) - arguments = tuple(arguments) - for k, s in given_kwargs.items(): - check_strategy(s, name=k) - s.validate() + with ensure_free_stackframes(): + for k, s in given_kwargs.items(): + check_strategy(s, name=k) + s.validate() stuff = Stuff(selfy=selfy, args=arguments, kwargs=kwargs, given_kwargs=given_kwargs) - return arguments, kwargs, test_runner, stuff + return arguments, kwargs, stuff def skip_exceptions_to_reraise(): @@ -678,9 +701,38 @@ def new_given_signature(original_sig, given_kwargs): ) +def default_executor(data, function): + return function(data) + + +def get_executor(runner): + try: + execute_example = runner.execute_example + except AttributeError: + pass + else: + return lambda data, function: execute_example(partial(function, data)) + + if hasattr(runner, "setup_example") or hasattr(runner, "teardown_example"): + setup = getattr(runner, "setup_example", None) or (lambda: None) + teardown = getattr(runner, "teardown_example", None) or (lambda ex: None) + + def execute(data, function): + token = None + try: + token = setup() + return function(data) + finally: + teardown(token) + + return execute + + return default_executor + + class StateForActualGivenExecution: - def __init__(self, test_runner, stuff, test, settings, random, wrapped_test): - self.test_runner = test_runner + def __init__(self, stuff, test, settings, random, wrapped_test): + self.test_runner = get_executor(stuff.selfy) self.stuff = stuff self.settings = settings self.last_exception = None @@ -691,6 +743,7 @@ class StateForActualGivenExecution: self.is_find = getattr(wrapped_test, "_hypothesis_internal_is_find", False) self.wrapped_test = wrapped_test + self.xfail_example_reprs = set() self.test = test @@ -707,6 +760,7 @@ class StateForActualGivenExecution: def execute_once( self, data, + *, print_example=False, is_final=False, expected_failure=None, @@ -752,54 +806,65 @@ class StateForActualGivenExecution: def run(data): # Set up dynamic context needed by a single test run. - with local_settings(self.settings): - with deterministic_PRNG(): - with BuildContext(data, is_final=is_final) as context: - if self.stuff.selfy is not None: - data.hypothesis_runner = self.stuff.selfy - # Generate all arguments to the test function. - args = self.stuff.args - kwargs = dict(self.stuff.kwargs) - if example_kwargs is None: - a, kw, argslices = context.prep_args_kwargs_from_strategies( - (), self.stuff.given_kwargs - ) - assert not a, "strategies all moved to kwargs by now" - else: - kw = example_kwargs - argslices = {} - kwargs.update(kw) - if expected_failure is not None: - nonlocal text_repr - text_repr = repr_call(test, args, kwargs) - - if print_example or current_verbosity() >= Verbosity.verbose: - printer = RepresentationPrinter(context=context) - if print_example: - printer.text("Falsifying example:") - else: - printer.text("Trying example:") - - if self.print_given_args: - printer.text(" ") - printer.repr_call( - test.__name__, - args, - kwargs, - force_split=True, - arg_slices=argslices, - leading_comment=( - "# " + context.data.slice_comments[(0, 0)] - if (0, 0) in context.data.slice_comments - else None - ), - ) - report(printer.getvalue()) - return test(*args, **kwargs) + if self.stuff.selfy is not None: + data.hypothesis_runner = self.stuff.selfy + # Generate all arguments to the test function. + args = self.stuff.args + kwargs = dict(self.stuff.kwargs) + if example_kwargs is None: + a, kw, argslices = context.prep_args_kwargs_from_strategies( + (), self.stuff.given_kwargs + ) + assert not a, "strategies all moved to kwargs by now" + else: + kw = example_kwargs + argslices = {} + kwargs.update(kw) + if expected_failure is not None: + nonlocal text_repr + text_repr = repr_call(test, args, kwargs) + if text_repr in self.xfail_example_reprs: + warnings.warn( + f"We generated {text_repr}, which seems identical " + "to one of your `@example(...).xfail()` cases. " + "Revise the strategy to avoid this overlap?", + HypothesisWarning, + # Checked in test_generating_xfailed_examples_warns! + stacklevel=6, + ) - # Run the test function once, via the executor hook. - # In most cases this will delegate straight to `run(data)`. - result = self.test_runner(data, run) + if print_example or current_verbosity() >= Verbosity.verbose: + printer = RepresentationPrinter(context=context) + if print_example: + printer.text("Falsifying example:") + else: + printer.text("Trying example:") + + if self.print_given_args: + printer.text(" ") + printer.repr_call( + test.__name__, + args, + kwargs, + force_split=True, + arg_slices=argslices, + leading_comment=( + "# " + context.data.slice_comments[(0, 0)] + if (0, 0) in context.data.slice_comments + else None + ), + ) + report(printer.getvalue()) + return test(*args, **kwargs) + + # self.test_runner can include the execute_example method, or setup/teardown + # _example, so it's important to get the PRNG and build context in place first. + with local_settings(self.settings): + with deterministic_PRNG(): + with BuildContext(data, is_final=is_final) as context: + # Run the test function once, via the executor hook. + # In most cases this will delegate straight to `run(data)`. + result = self.test_runner(data, run) # If a failure was expected, it should have been raised already, so # instead raise an appropriate diagnostic error. @@ -879,7 +944,8 @@ class StateForActualGivenExecution: except ( HypothesisDeprecationWarning, FailedHealthCheck, - ) + skip_exceptions_to_reraise(): + *skip_exceptions_to_reraise(), + ): # These are fatal errors or control exceptions that should stop the # engine, so we re-raise them. raise @@ -1020,15 +1086,6 @@ class StateForActualGivenExecution: _raise_to_user(errors_to_report, self.settings, report_lines) -def add_note(exc, note): - try: - exc.add_note(note) - except AttributeError: - if not hasattr(exc, "__notes__"): - exc.__notes__ = [] - exc.__notes__.append(note) - - def _raise_to_user(errors_to_report, settings, target_lines, trailer=""): """Helper function for attaching notes and grouping multiple errors.""" failing_prefix = "Falsifying example: " @@ -1117,7 +1174,16 @@ class HypothesisHandle: @overload def given( - *_given_arguments: Union[SearchStrategy[Any], EllipsisType], + _: EllipsisType, / +) -> Callable[ + [Callable[..., Optional[Coroutine[Any, Any, None]]]], Callable[[], None] +]: # pragma: no cover + ... + + +@overload +def given( + *_given_arguments: SearchStrategy[Any], ) -> Callable[ [Callable[..., Optional[Coroutine[Any, Any, None]]]], Callable[..., None] ]: # pragma: no cover @@ -1200,6 +1266,8 @@ def given( ) given_kwargs[name] = st.from_type(hints[name]) + prev_self = Unset = object() + @impersonate(test) @define_function_signature(test.__name__, test.__doc__, new_signature) def wrapped_test(*arguments, **kwargs): @@ -1222,14 +1290,13 @@ def given( random = get_random_for_wrapped_test(test, wrapped_test) - processed_args = process_arguments_to_given( + arguments, kwargs, stuff = process_arguments_to_given( wrapped_test, arguments, kwargs, given_kwargs, new_signature.parameters ) - arguments, kwargs, test_runner, stuff = processed_args if ( inspect.iscoroutinefunction(test) - and test_runner is default_new_style_executor + and get_executor(stuff.selfy) is default_executor ): # See https://github.com/HypothesisWorks/hypothesis/issues/3054 # If our custom executor doesn't handle coroutines, or we return an @@ -1261,9 +1328,26 @@ def given( "to ensure that each example is run in a separate " "database transaction." ) + if settings.database is not None: + nonlocal prev_self + # Check selfy really is self (not e.g. a mock) before we health-check + cur_self = ( + stuff.selfy + if getattr(type(stuff.selfy), test.__name__, None) is wrapped_test + else None + ) + if prev_self is Unset: + prev_self = cur_self + elif cur_self is not prev_self: + msg = ( + f"The method {test.__qualname__} was called from multiple " + "different executors. This may lead to flaky tests and " + "nonreproducible errors when replaying from database." + ) + fail_health_check(settings, msg, HealthCheck.differing_executors) state = StateForActualGivenExecution( - test_runner, stuff, test, settings, random, wrapped_test + stuff, test, settings, random, wrapped_test ) reproduce_failure = wrapped_test._hypothesis_internal_use_reproduce_failure @@ -1406,13 +1490,13 @@ def given( parent=wrapped_test._hypothesis_internal_use_settings, deadline=None ) random = get_random_for_wrapped_test(test, wrapped_test) - _args, _kwargs, test_runner, stuff = process_arguments_to_given( + _args, _kwargs, stuff = process_arguments_to_given( wrapped_test, (), {}, given_kwargs, new_signature.parameters ) assert not _args assert not _kwargs state = StateForActualGivenExecution( - test_runner, stuff, test, settings, random, wrapped_test + stuff, test, settings, random, wrapped_test ) digest = function_digest(test) # We track the minimal-so-far example for each distinct origin, so @@ -1490,6 +1574,9 @@ def find( ) if database_key is None and settings.database is not None: + # Note: The database key is not guaranteed to be unique. If not, replaying + # of database examples may fail to reproduce due to being replayed on the + # wrong condition. database_key = function_digest(condition) if not isinstance(specifier, SearchStrategy): @@ -1506,7 +1593,7 @@ def find( def test(v): if condition(v): last[:] = [v] - raise Found() + raise Found if random is not None: test = seed(random.getrandbits(64))(test) diff --git a/contrib/python/hypothesis/py3/hypothesis/database.py b/contrib/python/hypothesis/py3/hypothesis/database.py index 7960a9d4e0..4abce10c01 100644 --- a/contrib/python/hypothesis/py3/hypothesis/database.py +++ b/contrib/python/hypothesis/py3/hypothesis/database.py @@ -15,6 +15,7 @@ import os import sys import warnings from datetime import datetime, timedelta, timezone +from functools import lru_cache from hashlib import sha384 from os import getenv from pathlib import Path, PurePath @@ -23,7 +24,7 @@ from urllib.error import HTTPError, URLError from urllib.request import Request, urlopen from zipfile import BadZipFile, ZipFile -from hypothesis.configuration import mkdir_p, storage_directory +from hypothesis.configuration import storage_directory from hypothesis.errors import HypothesisException, HypothesisWarning from hypothesis.utils.conventions import not_set @@ -37,16 +38,16 @@ __all__ = [ ] -def _usable_dir(path): +def _usable_dir(path: Path) -> bool: """ Returns True iff the desired path can be used as database path because either the directory exists and can be used, or its root directory can be used and we can make the directory as needed. """ - while not os.path.exists(path): + while not path.exists(): # Loop terminates because the root dir ('/' on unix) always exists. - path = os.path.dirname(path) - return os.path.isdir(path) and os.access(path, os.R_OK | os.W_OK | os.X_OK) + path = path.parent + return path.is_dir() and os.access(path, os.R_OK | os.W_OK | os.X_OK) def _db_for_path(path=None): @@ -61,11 +62,11 @@ def _db_for_path(path=None): path = storage_directory("examples") if not _usable_dir(path): # pragma: no cover warnings.warn( - HypothesisWarning( - "The database setting is not configured, and the default " - "location is unusable - falling back to an in-memory " - f"database for this session. path={path!r}" - ) + "The database setting is not configured, and the default " + "location is unusable - falling back to an in-memory " + f"database for this session. {path=}", + HypothesisWarning, + stacklevel=3, ) return InMemoryExampleDatabase() if path in (None, ":memory:"): @@ -188,33 +189,31 @@ class DirectoryBasedExampleDatabase(ExampleDatabase): the :class:`~hypothesis.database.MultiplexedDatabase` helper. """ - def __init__(self, path: str) -> None: - self.path = path - self.keypaths: Dict[bytes, str] = {} + def __init__(self, path: os.PathLike) -> None: + self.path = Path(path) + self.keypaths: Dict[bytes, Path] = {} def __repr__(self) -> str: return f"DirectoryBasedExampleDatabase({self.path!r})" - def _key_path(self, key: bytes) -> str: + def _key_path(self, key: bytes) -> Path: try: return self.keypaths[key] except KeyError: pass - directory = os.path.join(self.path, _hash(key)) - self.keypaths[key] = directory - return directory + self.keypaths[key] = self.path / _hash(key) + return self.keypaths[key] def _value_path(self, key, value): - return os.path.join(self._key_path(key), _hash(value)) + return self._key_path(key) / _hash(value) def fetch(self, key: bytes) -> Iterable[bytes]: kp = self._key_path(key) - if not os.path.exists(kp): + if not kp.is_dir(): return for path in os.listdir(kp): try: - with open(os.path.join(kp, path), "rb") as i: - yield i.read() + yield (kp / path).read_bytes() except OSError: pass @@ -222,18 +221,17 @@ class DirectoryBasedExampleDatabase(ExampleDatabase): # Note: we attempt to create the dir in question now. We # already checked for permissions, but there can still be other issues, # e.g. the disk is full - mkdir_p(self._key_path(key)) + self._key_path(key).mkdir(exist_ok=True, parents=True) path = self._value_path(key, value) - if not os.path.exists(path): + if not path.exists(): suffix = binascii.hexlify(os.urandom(16)).decode("ascii") - tmpname = path + "." + suffix - with open(tmpname, "wb") as o: - o.write(value) + tmpname = path.with_suffix(f"{path.suffix}.{suffix}") + tmpname.write_bytes(value) try: - os.rename(tmpname, path) + tmpname.rename(path) except OSError: # pragma: no cover - os.unlink(tmpname) - assert not os.path.exists(tmpname) + tmpname.unlink() + assert not tmpname.exists() def move(self, src: bytes, dest: bytes, value: bytes) -> None: if src == dest: @@ -250,7 +248,7 @@ class DirectoryBasedExampleDatabase(ExampleDatabase): def delete(self, key: bytes, value: bytes) -> None: try: - os.unlink(self._value_path(key, value)) + self._value_path(key, value).unlink() except OSError: pass @@ -413,15 +411,13 @@ class GitHubArtifactDatabase(ExampleDatabase): repo: str, artifact_name: str = "hypothesis-example-db", cache_timeout: timedelta = timedelta(days=1), - path: Optional[Path] = None, + path: Optional[os.PathLike] = None, ): self.owner = owner self.repo = repo self.artifact_name = artifact_name self.cache_timeout = cache_timeout - self.keypaths: Dict[bytes, PurePath] = {} - # Get the GitHub token from the environment # It's unnecessary to use a token if the repo is public self.token: Optional[str] = getenv("GITHUB_TOKEN") @@ -431,7 +427,7 @@ class GitHubArtifactDatabase(ExampleDatabase): storage_directory(f"github-artifacts/{self.artifact_name}/") ) else: - self.path = path + self.path = Path(path) # We don't want to initialize the cache until we need to self._initialized: bool = False @@ -488,11 +484,11 @@ class GitHubArtifactDatabase(ExampleDatabase): self._access_cache[keypath].add(PurePath(filename)) except BadZipFile: warnings.warn( - HypothesisWarning( - "The downloaded artifact from GitHub is invalid. " - "This could be because the artifact was corrupted, " - "or because the artifact was not created by Hypothesis. " - ) + "The downloaded artifact from GitHub is invalid. " + "This could be because the artifact was corrupted, " + "or because the artifact was not created by Hypothesis. ", + HypothesisWarning, + stacklevel=3, ) self._disabled = True @@ -500,7 +496,7 @@ class GitHubArtifactDatabase(ExampleDatabase): def _initialize_db(self) -> None: # Create the cache directory if it doesn't exist - mkdir_p(str(self.path)) + self.path.mkdir(exist_ok=True, parents=True) # Get all artifacts cached_artifacts = sorted( @@ -534,18 +530,17 @@ class GitHubArtifactDatabase(ExampleDatabase): self._artifact = new_artifact elif found_artifact is not None: warnings.warn( - HypothesisWarning( - "Using an expired artifact as a fallback for the database: " - f"{found_artifact}" - ) + "Using an expired artifact as a fallback for the database: " + f"{found_artifact}", + HypothesisWarning, + stacklevel=2, ) self._artifact = found_artifact else: warnings.warn( - HypothesisWarning( - "Couldn't acquire a new or existing artifact. " - "Disabling database." - ) + "Couldn't acquire a new or existing artifact. Disabling database.", + HypothesisWarning, + stacklevel=2, ) self._disabled = True return @@ -561,11 +556,11 @@ class GitHubArtifactDatabase(ExampleDatabase): "Authorization": f"Bearer {self.token}", }, ) + warning_message = None response_bytes: Optional[bytes] = None try: - response = urlopen(request) - response_bytes = response.read() - warning_message = None + with urlopen(request) as response: + response_bytes = response.read() except HTTPError as e: if e.code == 401: warning_message = ( @@ -587,7 +582,7 @@ class GitHubArtifactDatabase(ExampleDatabase): ) if warning_message is not None: - warnings.warn(HypothesisWarning(warning_message)) + warnings.warn(warning_message, HypothesisWarning, stacklevel=4) return None return response_bytes @@ -620,25 +615,21 @@ class GitHubArtifactDatabase(ExampleDatabase): timestamp = datetime.now(timezone.utc).isoformat().replace(":", "_") artifact_path = self.path / f"{timestamp}.zip" try: - with open(artifact_path, "wb") as f: - f.write(artifact_bytes) + artifact_path.write_bytes(artifact_bytes) except OSError: warnings.warn( - HypothesisWarning("Could not save the latest artifact from GitHub. ") + "Could not save the latest artifact from GitHub. ", + HypothesisWarning, + stacklevel=3, ) return None return artifact_path - def _key_path(self, key: bytes) -> PurePath: - try: - return self.keypaths[key] - except KeyError: - pass - - directory = PurePath(_hash(key) + "/") - self.keypaths[key] = directory - return directory + @staticmethod + @lru_cache + def _key_path(key: bytes) -> PurePath: + return PurePath(_hash(key) + "/") def fetch(self, key: bytes) -> Iterable[bytes]: if self._disabled: diff --git a/contrib/python/hypothesis/py3/hypothesis/entry_points.py b/contrib/python/hypothesis/py3/hypothesis/entry_points.py index e99ac5ffcc..82a010da42 100644 --- a/contrib/python/hypothesis/py3/hypothesis/entry_points.py +++ b/contrib/python/hypothesis/py3/hypothesis/entry_points.py @@ -15,51 +15,23 @@ custom types, running the relevant code when *hypothesis* is imported instead of your package. """ -try: - # We prefer to use importlib.metadata, or the backport on Python <= 3.7, - # because it's much faster than pkg_resources (200ms import time speedup). - try: - from importlib import metadata as importlib_metadata - except ImportError: - import importlib_metadata # type: ignore # mypy thinks this is a redefinition +import importlib.metadata +import os - def get_entry_points(): - try: - eps = importlib_metadata.entry_points(group="hypothesis") - except TypeError: # pragma: no cover - # Load-time selection requires Python >= 3.10 or importlib_metadata >= 3.6, - # so we'll retain this fallback logic for some time to come. See also - # https://importlib-metadata.readthedocs.io/en/latest/using.html - eps = importlib_metadata.entry_points().get("hypothesis", []) - yield from eps -except ImportError: - # But if we're not on Python >= 3.8 and the importlib_metadata backport - # is not installed, we fall back to pkg_resources anyway. +def get_entry_points(): try: - import pkg_resources - except ImportError: - import warnings - - from hypothesis.errors import HypothesisWarning - - warnings.warn( - "Under Python <= 3.7, Hypothesis requires either the importlib_metadata " - "or setuptools package in order to load plugins via entrypoints.", - HypothesisWarning, - ) - - def get_entry_points(): - yield from () - - else: - - def get_entry_points(): - yield from pkg_resources.iter_entry_points("hypothesis") + eps = importlib.metadata.entry_points(group="hypothesis") + except TypeError: # pragma: no cover + # Load-time selection requires Python >= 3.10. See also + # https://importlib-metadata.readthedocs.io/en/latest/using.html + eps = importlib.metadata.entry_points().get("hypothesis", []) + yield from eps def run(): - for entry in get_entry_points(): # pragma: no cover - hook = entry.load() - if callable(hook): - hook() + if not os.environ.get("HYPOTHESIS_NO_PLUGINS"): + for entry in get_entry_points(): # pragma: no cover + hook = entry.load() + if callable(hook): + hook() diff --git a/contrib/python/hypothesis/py3/hypothesis/errors.py b/contrib/python/hypothesis/py3/hypothesis/errors.py index 7b6de721fb..9ee81cfc36 100644 --- a/contrib/python/hypothesis/py3/hypothesis/errors.py +++ b/contrib/python/hypothesis/py3/hypothesis/errors.py @@ -129,6 +129,7 @@ def __getattr__(name): "instead, or `exceptiongroup.BaseExceptionGroup` before Python 3.11", since="2022-08-02", has_codemod=False, # This would be a great PR though! + stacklevel=1, ) return BaseExceptionGroup diff --git a/contrib/python/hypothesis/py3/hypothesis/executors.py b/contrib/python/hypothesis/py3/hypothesis/executors.py deleted file mode 100644 index 67895467e4..0000000000 --- a/contrib/python/hypothesis/py3/hypothesis/executors.py +++ /dev/null @@ -1,65 +0,0 @@ -# This file is part of Hypothesis, which may be found at -# https://github.com/HypothesisWorks/hypothesis/ -# -# Copyright the Hypothesis Authors. -# Individual contributors are listed in AUTHORS.rst and the git log. -# -# This Source Code Form is subject to the terms of the Mozilla Public License, -# v. 2.0. If a copy of the MPL was not distributed with this file, You can -# obtain one at https://mozilla.org/MPL/2.0/. - - -def default_executor(function): # pragma: nocover - raise NotImplementedError() # We don't actually use this any more - - -def setup_teardown_executor(setup, teardown): - setup = setup or (lambda: None) - teardown = teardown or (lambda ex: None) - - def execute(function): - token = None - try: - token = setup() - return function() - finally: - teardown(token) - - return execute - - -def executor(runner): - try: - return runner.execute_example - except AttributeError: - pass - - if hasattr(runner, "setup_example") or hasattr(runner, "teardown_example"): - return setup_teardown_executor( - getattr(runner, "setup_example", None), - getattr(runner, "teardown_example", None), - ) - - return default_executor - - -def default_new_style_executor(data, function): - return function(data) - - -class ConjectureRunner: - def hypothesis_execute_example_with_data(self, data, function): - return function(data) - - -def new_style_executor(runner): - if runner is None: - return default_new_style_executor - if isinstance(runner, ConjectureRunner): - return runner.hypothesis_execute_example_with_data - - old_school = executor(runner) - if old_school is default_executor: - return default_new_style_executor - else: - return lambda data, function: old_school(lambda: function(data)) diff --git a/contrib/python/hypothesis/py3/hypothesis/extra/_array_helpers.py b/contrib/python/hypothesis/py3/hypothesis/extra/_array_helpers.py index 5e6718eea8..a525d3c473 100644 --- a/contrib/python/hypothesis/py3/hypothesis/extra/_array_helpers.py +++ b/contrib/python/hypothesis/py3/hypothesis/extra/_array_helpers.py @@ -275,10 +275,10 @@ def broadcastable_shapes( # We are unsure if gufuncs allow frozen dimensions to be optional, but it's # easy enough to support here - and so we will unless we learn otherwise. _DIMENSION = r"\w+\??" # Note that \w permits digits too! -_SHAPE = r"\((?:{0}(?:,{0})".format(_DIMENSION) + r"{0,31})?\)" -_ARGUMENT_LIST = "{0}(?:,{0})*".format(_SHAPE) +_SHAPE = rf"\((?:{_DIMENSION}(?:,{_DIMENSION}){{0,31}})?\)" +_ARGUMENT_LIST = f"{_SHAPE}(?:,{_SHAPE})*" _SIGNATURE = rf"^{_ARGUMENT_LIST}->{_SHAPE}$" -_SIGNATURE_MULTIPLE_OUTPUT = r"^{0}->{0}$".format(_ARGUMENT_LIST) +_SIGNATURE_MULTIPLE_OUTPUT = rf"^{_ARGUMENT_LIST}->{_ARGUMENT_LIST}$" class _GUfuncSig(NamedTuple): @@ -286,7 +286,7 @@ class _GUfuncSig(NamedTuple): result_shape: Shape -def _hypothesis_parse_gufunc_signature(signature, all_checks=True): +def _hypothesis_parse_gufunc_signature(signature): # Disable all_checks to better match the Numpy version, for testing if not re.match(_SIGNATURE, signature): if re.match(_SIGNATURE_MULTIPLE_OUTPUT, signature): @@ -294,7 +294,7 @@ def _hypothesis_parse_gufunc_signature(signature, all_checks=True): "Hypothesis does not yet support generalised ufunc signatures " "with multiple output arrays - mostly because we don't know of " "anyone who uses them! Please get in touch with us to fix that." - f"\n (signature={signature!r})" + f"\n ({signature=})" ) if re.match( ( @@ -305,7 +305,7 @@ def _hypothesis_parse_gufunc_signature(signature, all_checks=True): signature, ): raise InvalidArgument( - f"signature={signature!r} matches Numpy's regex for gufunc signatures, " + f"{signature=} matches Numpy's regex for gufunc signatures, " f"but contains shapes with more than {NDIM_MAX} dimensions and is thus invalid." ) raise InvalidArgument(f"{signature!r} is not a valid gufunc signature") @@ -315,34 +315,29 @@ def _hypothesis_parse_gufunc_signature(signature, all_checks=True): ) assert len(output_shapes) == 1 result_shape = output_shapes[0] - if all_checks: - # Check that there are no names in output shape that do not appear in inputs. - # (kept out of parser function for easier generation of test values) - # We also disallow frozen optional dimensions - this is ambiguous as there is - # no way to share an un-named dimension between shapes. Maybe just padding? - # Anyway, we disallow it pending clarification from upstream. - frozen_optional_err = ( - "Got dimension %r, but handling of frozen optional dimensions " - "is ambiguous. If you known how this should work, please " - "contact us to get this fixed and documented (signature=%r)." - ) - only_out_err = ( - "The %r dimension only appears in the output shape, and is " - "not frozen, so the size is not determined (signature=%r)." - ) - names_in = {n.strip("?") for shp in input_shapes for n in shp} - names_out = {n.strip("?") for n in result_shape} - for shape in input_shapes + (result_shape,): - for name in shape: - try: - int(name.strip("?")) - if "?" in name: - raise InvalidArgument(frozen_optional_err % (name, signature)) - except ValueError: - if name.strip("?") in (names_out - names_in): - raise InvalidArgument( - only_out_err % (name, signature) - ) from None + # Check that there are no names in output shape that do not appear in inputs. + # (kept out of parser function for easier generation of test values) + # We also disallow frozen optional dimensions - this is ambiguous as there is + # no way to share an un-named dimension between shapes. Maybe just padding? + # Anyway, we disallow it pending clarification from upstream. + for shape in (*input_shapes, result_shape): + for name in shape: + try: + int(name.strip("?")) + if "?" in name: + raise InvalidArgument( + f"Got dimension {name!r}, but handling of frozen optional dimensions " + "is ambiguous. If you known how this should work, please " + "contact us to get this fixed and documented ({signature=})." + ) + except ValueError: + names_in = {n.strip("?") for shp in input_shapes for n in shp} + names_out = {n.strip("?") for n in result_shape} + if name.strip("?") in (names_out - names_in): + raise InvalidArgument( + "The {name!r} dimension only appears in the output shape, and is " + "not frozen, so the size is not determined ({signature=})." + ) from None return _GUfuncSig(input_shapes=input_shapes, result_shape=result_shape) @@ -408,7 +403,7 @@ def mutually_broadcastable_shapes( ) check_type(str, signature, "signature") parsed_signature = _hypothesis_parse_gufunc_signature(signature) - all_shapes = parsed_signature.input_shapes + (parsed_signature.result_shape,) + all_shapes = (*parsed_signature.input_shapes, parsed_signature.result_shape) sig_dims = min(len(s) for s in all_shapes) num_shapes = len(parsed_signature.input_shapes) @@ -540,7 +535,7 @@ class MutuallyBroadcastableShapesStrategy(st.SearchStrategy): # that we can do an accurate per-shape length cap. dims = {} shapes = [] - for shape in self.signature.input_shapes + (self.signature.result_shape,): + for shape in (*self.signature.input_shapes, self.signature.result_shape): shapes.append([]) for name in shape: if name.isdigit(): @@ -614,7 +609,7 @@ class MutuallyBroadcastableShapesStrategy(st.SearchStrategy): if not any(use): break - result_shape = result_shape[: max(map(len, [self.base_shape] + shapes))] + result_shape = result_shape[: max(map(len, [self.base_shape, *shapes]))] assert len(shapes) == self.num_shapes assert all(self.min_dims <= len(s) <= self.max_dims for s in shapes) diff --git a/contrib/python/hypothesis/py3/hypothesis/extra/_patching.py b/contrib/python/hypothesis/py3/hypothesis/extra/_patching.py index d63f14bc6b..49b15ebc37 100644 --- a/contrib/python/hypothesis/py3/hypothesis/extra/_patching.py +++ b/contrib/python/hypothesis/py3/hypothesis/extra/_patching.py @@ -33,7 +33,6 @@ from libcst import matchers as m from libcst.codemod import CodemodContext, VisitorBasedCodemodCommand from hypothesis.configuration import storage_directory -from hypothesis.extra.codemods import _native_parser from hypothesis.version import __version__ try: @@ -68,30 +67,17 @@ def indent(text: str, prefix: str) -> str: class AddExamplesCodemod(VisitorBasedCodemodCommand): DESCRIPTION = "Add explicit examples to failing tests." - @classmethod - def refactor(cls, code, fn_examples, *, strip_via=(), dec="example"): + def __init__(self, context, fn_examples, strip_via=(), dec="example", width=88): """Add @example() decorator(s) for failing test(s). `code` is the source code of the module where the test functions are defined. `fn_examples` is a dict of function name to list-of-failing-examples. """ - assert not isinstance(strip_via, str), "expected a collection of strings" - dedented, prefix = dedent(code) - with _native_parser(): - mod = cst.parse_module(dedented) - modded = ( - cls(CodemodContext(), fn_examples, prefix, strip_via, dec) - .transform_module(mod) - .code - ) - return indent(modded, prefix=prefix) - - def __init__(self, context, fn_examples, prefix="", strip_via=(), dec="example"): assert fn_examples, "This codemod does nothing without fn_examples." super().__init__(context) self.decorator_func = cst.parse_expression(dec) - self.line_length = 88 - len(prefix) # to match Black's default formatting + self.line_length = width value_in_strip_via = m.MatchIfTrue(lambda x: literal_eval(x.value) in strip_via) self.strip_matching = m.Call( m.Attribute(m.Call(), m.Name("via")), @@ -158,10 +144,14 @@ def get_patch_for(func, failing_examples, *, strip_via=()): # The printed examples might include object reprs which are invalid syntax, # so we parse here and skip over those. If _none_ are valid, there's no patch. call_nodes = [] - for ex, via in failing_examples: + for ex, via in set(failing_examples): with suppress(Exception): node = cst.parse_expression(ex) assert isinstance(node, cst.Call), node + # Check for st.data(), which doesn't support explicit examples + data = m.Arg(m.Call(m.Name("data"), args=[m.Arg(m.Ellipsis())])) + if m.matches(node, m.Call(args=[m.ZeroOrMore(), data, m.ZeroOrMore()])): + return None call_nodes.append((node, via)) if not call_nodes: return None @@ -175,13 +165,20 @@ def get_patch_for(func, failing_examples, *, strip_via=()): decorator_func = "example" # Do the codemod and return a triple containing location and replacement info. - after = AddExamplesCodemod.refactor( - before, + dedented, prefix = dedent(before) + try: + node = cst.parse_module(dedented) + except Exception: # pragma: no cover + # inspect.getsource() sometimes returns a decorator alone, which is invalid + return None + after = AddExamplesCodemod( + CodemodContext(), fn_examples={func.__name__: call_nodes}, strip_via=strip_via, dec=decorator_func, - ) - return (str(fname), before, after) + width=88 - len(prefix), # to match Black's default formatting + ).transform_module(node) + return (str(fname), before, indent(after.code, prefix=prefix)) def make_patch(triples, *, msg="Hypothesis: add explicit examples", when=None): @@ -209,7 +206,7 @@ def make_patch(triples, *, msg="Hypothesis: add explicit examples", when=None): def save_patch(patch: str, *, slug: str = "") -> Path: # pragma: no cover - assert re.fullmatch(r"|[a-z]+-", slug), f"malformed slug={slug!r}" + assert re.fullmatch(r"|[a-z]+-", slug), f"malformed {slug=}" now = date.today().isoformat() cleaned = re.sub(r"^Date: .+?$", "", patch, count=1, flags=re.MULTILINE) hash8 = hashlib.sha1(cleaned.encode()).hexdigest()[:8] diff --git a/contrib/python/hypothesis/py3/hypothesis/extra/array_api.py b/contrib/python/hypothesis/py3/hypothesis/extra/array_api.py index 63c71aa19d..cb5a3fee40 100644 --- a/contrib/python/hypothesis/py3/hypothesis/extra/array_api.py +++ b/contrib/python/hypothesis/py3/hypothesis/extra/array_api.py @@ -8,15 +8,12 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -import sys - -if sys.version_info[:2] < (3, 8): - raise RuntimeError("The Array API standard requires Python 3.8 or later") - import math +import sys from numbers import Real from types import SimpleNamespace from typing import ( + TYPE_CHECKING, Any, Iterable, Iterator, @@ -64,13 +61,16 @@ from hypothesis.internal.validation import ( from hypothesis.strategies._internal.strategies import check_strategy from hypothesis.strategies._internal.utils import defines_strategy +if TYPE_CHECKING: + from typing import TypeAlias + __all__ = [ "make_strategies_namespace", ] RELEASED_VERSIONS = ("2021.12", "2022.12") -NOMINAL_VERSIONS = RELEASED_VERSIONS + ("draft",) +NOMINAL_VERSIONS = (*RELEASED_VERSIONS, "draft") assert sorted(NOMINAL_VERSIONS) == list(NOMINAL_VERSIONS) # sanity check NominalVersion = Literal["2021.12", "2022.12", "draft"] assert get_args(NominalVersion) == NOMINAL_VERSIONS # sanity check @@ -83,7 +83,7 @@ FLOAT_NAMES = ("float32", "float64") REAL_NAMES = ALL_INT_NAMES + FLOAT_NAMES COMPLEX_NAMES = ("complex64", "complex128") NUMERIC_NAMES = REAL_NAMES + COMPLEX_NAMES -DTYPE_NAMES = ("bool",) + NUMERIC_NAMES +DTYPE_NAMES = ("bool", *NUMERIC_NAMES) DataType = TypeVar("DataType") @@ -118,6 +118,7 @@ def warn_on_missing_dtypes(xp: Any, stubs: List[str]) -> None: f"Array module {xp.__name__} does not have the following " f"dtypes in its namespace: {f_stubs}", HypothesisWarning, + stacklevel=3, ) @@ -454,9 +455,7 @@ class ArrayStrategy(st.SearchStrategy): else: self.check_set_value(val, val_0d, self.elements_strategy) - result = self.xp.reshape(result, self.shape) - - return result + return self.xp.reshape(result, self.shape) def _arrays( @@ -563,7 +562,7 @@ def _arrays( raise InvalidArgument(f"shape={shape} is not a valid shape or strategy") check_argument( all(isinstance(x, int) and x >= 0 for x in shape), - f"shape={shape!r}, but all dimensions must be non-negative integers.", + f"{shape=}, but all dimensions must be non-negative integers.", ) if elements is None: @@ -659,8 +658,13 @@ def numeric_dtype_names(base_name: str, sizes: Sequence[int]) -> Iterator[str]: yield f"{base_name}{size}" +IntSize: "TypeAlias" = Literal[8, 16, 32, 64] +FltSize: "TypeAlias" = Literal[32, 64] +CpxSize: "TypeAlias" = Literal[64, 128] + + def _integer_dtypes( - xp: Any, *, sizes: Union[int, Sequence[int]] = (8, 16, 32, 64) + xp: Any, *, sizes: Union[IntSize, Sequence[IntSize]] = (8, 16, 32, 64) ) -> st.SearchStrategy[DataType]: """Return a strategy for signed integer dtype objects. @@ -678,7 +682,7 @@ def _integer_dtypes( def _unsigned_integer_dtypes( - xp: Any, *, sizes: Union[int, Sequence[int]] = (8, 16, 32, 64) + xp: Any, *, sizes: Union[IntSize, Sequence[IntSize]] = (8, 16, 32, 64) ) -> st.SearchStrategy[DataType]: """Return a strategy for unsigned integer dtype objects. @@ -698,7 +702,7 @@ def _unsigned_integer_dtypes( def _floating_dtypes( - xp: Any, *, sizes: Union[int, Sequence[int]] = (32, 64) + xp: Any, *, sizes: Union[FltSize, Sequence[FltSize]] = (32, 64) ) -> st.SearchStrategy[DataType]: """Return a strategy for real-valued floating-point dtype objects. @@ -716,7 +720,7 @@ def _floating_dtypes( def _complex_dtypes( - xp: Any, *, sizes: Union[int, Sequence[int]] = (64, 128) + xp: Any, *, sizes: Union[CpxSize, Sequence[CpxSize]] = (64, 128) ) -> st.SearchStrategy[DataType]: """Return a strategy for complex dtype objects. @@ -802,7 +806,7 @@ def indices( check_type(tuple, shape, "shape") check_argument( all(isinstance(x, int) and x >= 0 for x in shape), - f"shape={shape!r}, but all dimensions must be non-negative integers.", + f"{shape=}, but all dimensions must be non-negative integers.", ) check_type(bool, allow_newaxis, "allow_newaxis") check_type(bool, allow_ellipsis, "allow_ellipsis") @@ -896,7 +900,7 @@ def make_strategies_namespace( check_argument( isinstance(xp.__array_api_version__, str) and xp.__array_api_version__ in RELEASED_VERSIONS, - f"xp.__array_api_version__={xp.__array_api_version__!r}, but it must " + f"{xp.__array_api_version__=}, but it must " f"be a valid version string {RELEASED_VERSIONS}. {not_available_msg}", ) api_version = xp.__array_api_version__ @@ -904,7 +908,7 @@ def make_strategies_namespace( else: check_argument( isinstance(api_version, str) and api_version in NOMINAL_VERSIONS, - f"api_version={api_version!r}, but it must be None, or a valid version " + f"{api_version=}, but it must be None, or a valid version " f"string in {RELEASED_VERSIONS}. {not_available_msg}", ) inferred_version = False @@ -915,6 +919,7 @@ def make_strategies_namespace( warn( f"Could not determine whether module {xp.__name__} is an Array API library", HypothesisWarning, + stacklevel=2, ) try: @@ -938,7 +943,7 @@ def make_strategies_namespace( ) -> st.SearchStrategy[Union[bool, int, float, complex]]: return _from_dtype( xp, - api_version, # type: ignore[arg-type] + api_version, dtype, min_value=min_value, max_value=max_value, @@ -962,7 +967,7 @@ def make_strategies_namespace( ) -> st.SearchStrategy: return _arrays( xp, - api_version, # type: ignore[arg-type] + api_version, dtype, shape, elements=elements, @@ -972,7 +977,7 @@ def make_strategies_namespace( @defines_strategy() def scalar_dtypes() -> st.SearchStrategy[DataType]: - return _scalar_dtypes(xp, api_version) # type: ignore[arg-type] + return _scalar_dtypes(xp, api_version) @defines_strategy() def boolean_dtypes() -> st.SearchStrategy[DataType]: @@ -984,23 +989,23 @@ def make_strategies_namespace( @defines_strategy() def numeric_dtypes() -> st.SearchStrategy[DataType]: - return _numeric_dtypes(xp, api_version) # type: ignore[arg-type] + return _numeric_dtypes(xp, api_version) @defines_strategy() def integer_dtypes( - *, sizes: Union[int, Sequence[int]] = (8, 16, 32, 64) + *, sizes: Union[IntSize, Sequence[IntSize]] = (8, 16, 32, 64) ) -> st.SearchStrategy[DataType]: return _integer_dtypes(xp, sizes=sizes) @defines_strategy() def unsigned_integer_dtypes( - *, sizes: Union[int, Sequence[int]] = (8, 16, 32, 64) + *, sizes: Union[IntSize, Sequence[IntSize]] = (8, 16, 32, 64) ) -> st.SearchStrategy[DataType]: return _unsigned_integer_dtypes(xp, sizes=sizes) @defines_strategy() def floating_dtypes( - *, sizes: Union[int, Sequence[int]] = (32, 64) + *, sizes: Union[FltSize, Sequence[FltSize]] = (32, 64) ) -> st.SearchStrategy[DataType]: return _floating_dtypes(xp, sizes=sizes) @@ -1017,7 +1022,7 @@ def make_strategies_namespace( class StrategiesNamespace(SimpleNamespace): def __init__(self, **kwargs): for attr in ["name", "api_version"]: - if attr not in kwargs.keys(): + if attr not in kwargs: raise ValueError(f"'{attr}' kwarg required") super().__init__(**kwargs) @@ -1061,7 +1066,7 @@ def make_strategies_namespace( @defines_strategy() def complex_dtypes( - *, sizes: Union[int, Sequence[int]] = (64, 128) + *, sizes: Union[CpxSize, Sequence[CpxSize]] = (64, 128) ) -> st.SearchStrategy[DataType]: return _complex_dtypes(xp, sizes=sizes) diff --git a/contrib/python/hypothesis/py3/hypothesis/extra/cli.py b/contrib/python/hypothesis/py3/hypothesis/extra/cli.py index 13afc1a85e..2e5ea5265e 100644 --- a/contrib/python/hypothesis/py3/hypothesis/extra/cli.py +++ b/contrib/python/hypothesis/py3/hypothesis/extra/cli.py @@ -43,6 +43,7 @@ import types from difflib import get_close_matches from functools import partial from multiprocessing import Pool +from pathlib import Path try: import pytest @@ -53,7 +54,7 @@ MESSAGE = """ The Hypothesis command-line interface requires the `{}` package, which you do not have installed. Run: - python -m pip install --upgrade hypothesis[cli] + python -m pip install --upgrade 'hypothesis[cli]' and try again. """ @@ -161,15 +162,28 @@ else: def _refactor(func, fname): try: - with open(fname) as f: - oldcode = f.read() + oldcode = Path(fname).read_text(encoding="utf-8") except (OSError, UnicodeError) as err: # Permissions or encoding issue, or file deleted, etc. return f"skipping {fname!r} due to {err}" - newcode = func(oldcode) + + if "hypothesis" not in oldcode: + return # This is a fast way to avoid running slow no-op codemods + + try: + newcode = func(oldcode) + except Exception as err: + from libcst import ParserSyntaxError + + if isinstance(err, ParserSyntaxError): + from hypothesis.extra._patching import indent + + msg = indent(str(err).replace("\n\n", "\n"), " ").strip() + return f"skipping {fname!r} due to {msg}" + raise + if newcode != oldcode: - with open(fname, mode="w") as f: - f.write(newcode) + Path(fname).write_text(newcode, encoding="utf-8") @main.command() # type: ignore # Click adds the .command attribute @click.argument("path", type=str, required=True, nargs=-1) @@ -275,9 +289,7 @@ else: help="force ghostwritten tests to be type-annotated (or not). " "By default, match the code to test.", ) - def write( - func, writer, except_, style, annotate - ): # noqa: D301 # \b disables autowrap + def write(func, writer, except_, style, annotate): # \b disables autowrap """`hypothesis write` writes property-based tests for you! Type annotations are helpful but not required for our advanced introspection diff --git a/contrib/python/hypothesis/py3/hypothesis/extra/codemods.py b/contrib/python/hypothesis/py3/hypothesis/extra/codemods.py index b5dcae72bb..b2828c31c3 100644 --- a/contrib/python/hypothesis/py3/hypothesis/extra/codemods.py +++ b/contrib/python/hypothesis/py3/hypothesis/extra/codemods.py @@ -47,30 +47,14 @@ at the cost of additional configuration (adding ``'hypothesis.extra'`` to the import functools import importlib -import os -from contextlib import contextmanager from inspect import Parameter, signature -from typing import List +from typing import ClassVar, List import libcst as cst import libcst.matchers as m from libcst.codemod import VisitorBasedCodemodCommand -@contextmanager -def _native_parser(): - # Only the native parser supports Python 3.9 and later, but for now it's - # only active if you set an environment variable. Very well then: - var = os.environ.get("LIBCST_PARSER_TYPE") - try: - os.environ["LIBCST_PARSER_TYPE"] = "native" - yield - finally: - os.environ.pop("LIBCST_PARSER_TYPE") - if var is not None: # pragma: no cover - os.environ["LIBCST_PARSER_TYPE"] = var - - def refactor(code: str) -> str: """Update a source code string from deprecated to modern Hypothesis APIs. @@ -80,11 +64,12 @@ def refactor(code: str) -> str: We recommend using the CLI, but if you want a Python function here it is. """ context = cst.codemod.CodemodContext() - with _native_parser(): - mod = cst.parse_module(code) + mod = cst.parse_module(code) transforms: List[VisitorBasedCodemodCommand] = [ HypothesisFixPositionalKeywonlyArgs(context), HypothesisFixComplexMinMagnitude(context), + HypothesisFixHealthcheckAll(context), + HypothesisFixCharactersArguments(context), ] for transform in transforms: mod = transform.transform_module(mod) @@ -126,7 +111,7 @@ class HypothesisFixComplexMinMagnitude(VisitorBasedCodemodCommand): return updated_node -@functools.lru_cache() +@functools.lru_cache def get_fn(import_path): mod, fn = import_path.rsplit(".", 1) return getattr(importlib.import_module(mod), fn) @@ -252,3 +237,48 @@ class HypothesisFixHealthcheckAll(VisitorBasedCodemodCommand): func=cst.Name("list"), args=[cst.Arg(value=cst.Name("Healthcheck"))], ) + + +class HypothesisFixCharactersArguments(VisitorBasedCodemodCommand): + """Fix deprecated white/blacklist arguments to characters:: + + st.characters(whitelist_categories=...) -> st.characters(categories=...) + st.characters(blacklist_categories=...) -> st.characters(exclude_categories=...) + st.characters(whitelist_characters=...) -> st.characters(include_characters=...) + st.characters(blacklist_characters=...) -> st.characters(exclude_characters=...) + + Additionally, we drop `exclude_categories=` if `categories=` is present, + because this argument is always redundant (or an error). + """ + + DESCRIPTION = "Fix deprecated white/blacklist arguments to characters." + METADATA_DEPENDENCIES = (cst.metadata.QualifiedNameProvider,) + + _replacements: ClassVar = { + "whitelist_categories": "categories", + "blacklist_categories": "exclude_categories", + "whitelist_characters": "include_characters", + "blacklist_characters": "exclude_characters", + } + + @m.leave( + m.Call( + metadata=match_qualname("hypothesis.strategies.characters"), + args=[ + m.ZeroOrMore(), + m.Arg(keyword=m.OneOf(*map(m.Name, _replacements))), + m.ZeroOrMore(), + ], + ), + ) + def fn(self, original_node, updated_node): + # Update to the new names + newargs = [] + for arg in updated_node.args: + kw = self._replacements.get(arg.keyword.value, arg.keyword.value) + newargs.append(arg.with_changes(keyword=cst.Name(kw))) + # Drop redundant exclude_categories, which is now an error + if any(m.matches(arg, m.Arg(keyword=m.Name("categories"))) for arg in newargs): + ex = m.Arg(keyword=m.Name("exclude_categories")) + newargs = [a for a in newargs if m.matches(a, ~ex)] + return updated_node.with_changes(args=newargs) diff --git a/contrib/python/hypothesis/py3/hypothesis/extra/django/_fields.py b/contrib/python/hypothesis/py3/hypothesis/extra/django/_fields.py index 2118cd406a..b4a574d69d 100644 --- a/contrib/python/hypothesis/py3/hypothesis/extra/django/_fields.py +++ b/contrib/python/hypothesis/py3/hypothesis/extra/django/_fields.py @@ -53,7 +53,7 @@ def integers_for_field(min_value, max_value): return inner -@lru_cache() +@lru_cache def timezones(): # From Django 4.0, the default is to use zoneinfo instead of pytz. assert getattr(django.conf.settings, "USE_TZ", False) @@ -186,7 +186,7 @@ def _for_form_ip(field): return st.ip_addresses(v=4).map(str) if validate_ipv6_address in field.default_validators: return _ipv6_strings - raise ResolutionFailed(f"No IP version validator on field={field!r}") + raise ResolutionFailed(f"No IP version validator on {field=}") @register_for(dm.DecimalField) @@ -246,9 +246,7 @@ def _for_text(field): # If there are no (usable) regexes, we use a standard text strategy. min_size, max_size = length_bounds_from_validators(field) strategy = st.text( - alphabet=st.characters( - blacklist_characters="\x00", blacklist_categories=("Cs",) - ), + alphabet=st.characters(exclude_characters="\x00", exclude_categories=("Cs",)), min_size=min_size, max_size=max_size, ).filter(lambda s: min_size <= len(s.strip())) @@ -275,11 +273,11 @@ def register_field_strategy( ``strategy`` must be a :class:`~hypothesis.strategies.SearchStrategy`. """ if not issubclass(field_type, (dm.Field, df.Field)): - raise InvalidArgument(f"field_type={field_type!r} must be a subtype of Field") + raise InvalidArgument(f"{field_type=} must be a subtype of Field") check_type(st.SearchStrategy, strategy, "strategy") if field_type in _global_field_lookup: raise InvalidArgument( - f"field_type={field_type!r} already has a registered " + f"{field_type=} already has a registered " f"strategy ({_global_field_lookup[field_type]!r})" ) if issubclass(field_type, dm.AutoField): diff --git a/contrib/python/hypothesis/py3/hypothesis/extra/django/_impl.py b/contrib/python/hypothesis/py3/hypothesis/extra/django/_impl.py index 8b461a6223..fb368fb566 100644 --- a/contrib/python/hypothesis/py3/hypothesis/extra/django/_impl.py +++ b/contrib/python/hypothesis/py3/hypothesis/extra/django/_impl.py @@ -11,8 +11,7 @@ import sys import unittest from functools import partial -from inspect import Parameter, signature -from typing import TYPE_CHECKING, Optional, Type, Union +from typing import TYPE_CHECKING, Optional, Type, TypeVar, Union from django import forms as df, test as dt from django.contrib.staticfiles import testing as dst @@ -22,7 +21,6 @@ from django.db import IntegrityError, models as dm from hypothesis import reject, strategies as st from hypothesis.errors import InvalidArgument from hypothesis.extra.django._fields import from_field -from hypothesis.internal.reflection import define_function_signature from hypothesis.strategies._internal.utils import defines_strategy if sys.version_info >= (3, 10): @@ -32,6 +30,8 @@ elif TYPE_CHECKING: else: EllipsisType = type(Ellipsis) +ModelT = TypeVar("ModelT", bound=dm.Model) + class HypothesisTestCase: def setup_example(self): @@ -66,8 +66,8 @@ class StaticLiveServerTestCase(HypothesisTestCase, dst.StaticLiveServerTestCase) @defines_strategy() def from_model( - *model: Type[dm.Model], **field_strategies: Union[st.SearchStrategy, EllipsisType] -) -> st.SearchStrategy: + model: Type[ModelT], /, **field_strategies: Union[st.SearchStrategy, EllipsisType] +) -> st.SearchStrategy[ModelT]: """Return a strategy for examples of ``model``. .. warning:: @@ -93,17 +93,10 @@ def from_model( ``...`` (:obj:`python:Ellipsis`) as a keyword argument to infer a strategy for a field which has a default value instead of using the default. """ - if len(model) == 1: - m_type = model[0] - elif len(model) > 1: - raise TypeError("Too many positional arguments") - else: - raise TypeError("Missing required positional argument `model`") - - if not issubclass(m_type, dm.Model): - raise InvalidArgument(f"model={model!r} must be a subtype of Model") + if not issubclass(model, dm.Model): + raise InvalidArgument(f"{model=} must be a subtype of Model") - fields_by_name = {f.name: f for f in m_type._meta.concrete_fields} + fields_by_name = {f.name: f for f in model._meta.concrete_fields} for name, value in sorted(field_strategies.items()): if value is ...: field_strategies[name] = from_field(fields_by_name[name]) @@ -116,31 +109,18 @@ def from_model( field_strategies[name] = from_field(field) for field in field_strategies: - if m_type._meta.get_field(field).primary_key: + if model._meta.get_field(field).primary_key: # The primary key is generated as part of the strategy. We # want to find any existing row with this primary key and # overwrite its contents. kwargs = {field: field_strategies.pop(field)} kwargs["defaults"] = st.fixed_dictionaries(field_strategies) # type: ignore - return _models_impl(st.builds(m_type.objects.update_or_create, **kwargs)) + return _models_impl(st.builds(model.objects.update_or_create, **kwargs)) # The primary key is not generated as part of the strategy, so we # just match against any row that has the same value for all # fields. - return _models_impl(st.builds(m_type.objects.get_or_create, **field_strategies)) - - -if sys.version_info[:2] >= (3, 8): - # See notes above definition of st.builds() - this signature is compatible - # and better matches the semantics of the function. Great for documentation! - sig = signature(from_model) - params = list(sig.parameters.values()) - params[0] = params[0].replace(kind=Parameter.POSITIONAL_ONLY) - from_model = define_function_signature( - name=from_model.__name__, - docstring=from_model.__doc__, - signature=sig.replace(parameters=params), - )(from_model) + return _models_impl(st.builds(model.objects.get_or_create, **field_strategies)) @st.composite @@ -187,7 +167,7 @@ def from_form( # ImageField form_kwargs = form_kwargs or {} if not issubclass(form, df.BaseForm): - raise InvalidArgument(f"form={form!r} must be a subtype of Form") + raise InvalidArgument(f"{form=} must be a subtype of Form") # Forms are a little bit different from models. Model classes have # all their fields defined, whereas forms may have different fields diff --git a/contrib/python/hypothesis/py3/hypothesis/extra/ghostwriter.py b/contrib/python/hypothesis/py3/hypothesis/extra/ghostwriter.py index f3590266ba..fde70caf75 100644 --- a/contrib/python/hypothesis/py3/hypothesis/extra/ghostwriter.py +++ b/contrib/python/hypothesis/py3/hypothesis/extra/ghostwriter.py @@ -83,7 +83,7 @@ import types import warnings from collections import OrderedDict, defaultdict from itertools import permutations, zip_longest -from keyword import iskeyword +from keyword import iskeyword as _iskeyword from string import ascii_lowercase from textwrap import dedent, indent from typing import ( @@ -103,18 +103,21 @@ from typing import ( Type, TypeVar, Union, + get_args, + get_origin, ) import black from hypothesis import Verbosity, find, settings, strategies as st from hypothesis.errors import InvalidArgument, SmallSearchSpaceWarning -from hypothesis.internal.compat import get_args, get_origin, get_type_hints +from hypothesis.internal.compat import get_type_hints from hypothesis.internal.reflection import get_signature, is_mock from hypothesis.internal.validation import check_type from hypothesis.provisional import domains from hypothesis.strategies._internal.collections import ListStrategy from hypothesis.strategies._internal.core import BuildsStrategy +from hypothesis.strategies._internal.deferred import DeferredStrategy from hypothesis.strategies._internal.flatmapped import FlatMapStrategy from hypothesis.strategies._internal.lazy import LazyStrategy, unwrap_strategies from hypothesis.strategies._internal.strategies import ( @@ -155,7 +158,6 @@ except {exceptions}: Except = Union[Type[Exception], Tuple[Type[Exception], ...]] ImportSet = Set[Union[str, Tuple[str, str]]] -RE_TYPES = (type(re.compile(".")), type(re.match(".", "abc"))) _quietly_settings = settings( database=None, deadline=None, @@ -259,6 +261,13 @@ def _type_from_doc_fragment(token: str) -> Optional[type]: return getattr(sys.modules.get(mod, None), name, None) +def _strip_typevars(type_): + with contextlib.suppress(Exception): + if {type(a) for a in get_args(type_)} == {TypeVar}: + return get_origin(type_) + return type_ + + def _strategy_for(param: inspect.Parameter, docstring: str) -> st.SearchStrategy: # Example types in docstrings: # - `:type a: sequence of integers` @@ -294,7 +303,7 @@ def _strategy_for(param: inspect.Parameter, docstring: str) -> st.SearchStrategy t = _type_from_doc_fragment(token) if isinstance(t, type) or is_generic_type(t): assert t is not None - types.append(t) + types.append(_strip_typevars(t)) if ( param.default is not inspect.Parameter.empty and param.default not in elements @@ -425,10 +434,9 @@ def _guess_strategy_by_argname(name: str) -> st.SearchStrategy: return st.nothing() if ( - name.endswith("_name") + name.endswith(("_name", "label")) or (name.endswith("name") and "_" not in name) or ("string" in name and "as" not in name) - or name.endswith("label") or name in STRING_NAMES ): return st.text() @@ -473,7 +481,7 @@ def _get_params(func: Callable) -> Dict[str, inspect.Parameter]: if arg.startswith("*") or arg == "...": kind = inspect.Parameter.KEYWORD_ONLY continue # we omit *varargs, if there are any - if iskeyword(arg.lstrip("*")) or not arg.lstrip("*").isidentifier(): + if _iskeyword(arg.lstrip("*")) or not arg.lstrip("*").isidentifier(): print(repr(args)) break # skip all subsequent params if this name is invalid params.append(inspect.Parameter(name=arg, kind=kind)) @@ -578,7 +586,7 @@ def _assert_eq(style: str, a: str, b: str) -> str: def _imports_for_object(obj): """Return the imports for `obj`, which may be empty for e.g. lambdas""" - if isinstance(obj, RE_TYPES): + if isinstance(obj, (re.Pattern, re.Match)): return {"re"} try: if is_generic_type(obj): @@ -659,6 +667,8 @@ def _valid_syntax_repr(strategy): # Flatten and de-duplicate any one_of strategies, whether that's from resolving # a Union type or combining inputs to multiple functions. try: + if isinstance(strategy, DeferredStrategy): + strategy = strategy.wrapped_strategy if isinstance(strategy, OneOfStrategy): seen = set() elems = [] @@ -672,6 +682,15 @@ def _valid_syntax_repr(strategy): # Trivial special case because the wrapped repr for text() is terrible. if strategy == st.text().wrapped_strategy: return set(), "text()" + # Remove any typevars; we don't exploit them so they're just clutter here + if ( + isinstance(strategy, LazyStrategy) + and strategy.function.__name__ == st.from_type.__name__ + and strategy._LazyStrategy__representation is None + ): + strategy._LazyStrategy__args = tuple( + _strip_typevars(a) for a in strategy._LazyStrategy__args + ) # Return a syntactically-valid strategy repr, including fixing some # strategy reprs and replacing invalid syntax reprs with `"nothing()"`. # String-replace to hide the special case in from_type() for Decimal('snan') @@ -698,6 +717,11 @@ def _get_module_helper(obj): # The goal is to show location from which obj should usually be accessed, rather # than what we assume is an internal submodule which defined it. module_name = obj.__module__ + + # if "collections.abc" is used don't use the deprecated aliases in "collections" + if module_name == "collections.abc": + return module_name + dots = [i for i, c in enumerate(module_name) if c == "."] + [None] for idx in dots: if getattr(sys.modules.get(module_name[:idx]), obj.__name__, None) is obj: @@ -721,7 +745,7 @@ def _get_module(obj): raise RuntimeError(f"Could not find module for ufunc {obj.__name__} ({obj!r}") -def _get_qualname(obj, include_module=False): +def _get_qualname(obj, *, include_module=False): # Replacing angle-brackets for objects defined in `.<locals>.` qname = getattr(obj, "__qualname__", obj.__name__) qname = qname.replace("<", "_").replace(">", "_").replace(" ", "") @@ -771,7 +795,7 @@ def _st_strategy_names(s: str) -> str: sets() too. """ names = "|".join(sorted(st.__all__, key=len, reverse=True)) - return re.sub(pattern=rf"\b(?:{names})\(", repl=r"st.\g<0>", string=s) + return re.sub(pattern=rf"\b(?:{names})\b[^= ]", repl=r"st.\g<0>", string=s) def _make_test_body( @@ -795,7 +819,7 @@ def _make_test_body( given_strategies = given_strategies or _get_strategies( *funcs, pass_result_to_next_func=ghost in ("idempotent", "roundtrip") ) - reprs = [((k,) + _valid_syntax_repr(v)) for k, v in given_strategies.items()] + reprs = [((k, *_valid_syntax_repr(v))) for k, v in given_strategies.items()] imports = imports.union(*(imp for _, imp, _ in reprs)) given_args = ", ".join(f"{k}={v}" for k, _, v in reprs) given_args = _st_strategy_names(given_args) @@ -963,14 +987,14 @@ def _parameter_to_annotation(parameter: Any) -> Optional[_AnnotationData]: set(), ) - type_name = f"{parameter.__module__}.{parameter.__name__}" + type_name = _get_qualname(parameter, include_module=True) # the types.UnionType does not support type arguments and needs to be translated if type_name == "types.UnionType": return _AnnotationData("typing.Union", {"typing"}) else: if hasattr(parameter, "__module__") and hasattr(parameter, "__name__"): - type_name = f"{parameter.__module__}.{parameter.__name__}" + type_name = _get_qualname(parameter, include_module=True) else: type_name = str(parameter) @@ -981,6 +1005,8 @@ def _parameter_to_annotation(parameter: Any) -> Optional[_AnnotationData]: return _AnnotationData(type_name, set(type_name.rsplit(".", maxsplit=1)[:-1])) arg_types = get_args(parameter) + if {type(a) for a in arg_types} == {TypeVar}: + arg_types = () # typing types get translated to classes that don't support generics origin_annotation: Optional[_AnnotationData] @@ -1017,13 +1043,14 @@ def _make_test(imports: ImportSet, body: str) -> str: # Discarding "builtins." and "__main__" probably isn't particularly useful # for user code, but important for making a good impression in demos. body = body.replace("builtins.", "").replace("__main__.", "") + body = body.replace("hypothesis.strategies.", "st.") if "st.from_type(typing." in body: imports.add("typing") imports |= {("hypothesis", "given"), ("hypothesis", "strategies as st")} if " reject()\n" in body: imports.add(("hypothesis", "reject")) - do_not_import = {"builtins", "__main__"} + do_not_import = {"builtins", "__main__", "hypothesis.strategies"} direct = {f"import {i}" for i in imports - do_not_import if isinstance(i, str)} from_imports = defaultdict(set) for module, name in {i for i in imports if isinstance(i, tuple)}: @@ -1305,7 +1332,7 @@ def fuzz( or :func:`~hypothesis.strategies.binary`. After that, you have a test! """ if not callable(func): - raise InvalidArgument(f"Got non-callable func={func!r}") + raise InvalidArgument(f"Got non-callable {func=}") except_ = _check_except(except_) _check_style(style) @@ -1365,7 +1392,7 @@ def idempotent( assert result == repeat, (result, repeat) """ if not callable(func): - raise InvalidArgument(f"Got non-callable func={func!r}") + raise InvalidArgument(f"Got non-callable {func=}") except_ = _check_except(except_) _check_style(style) @@ -1604,14 +1631,14 @@ def binary_operation( ) """ if not callable(func): - raise InvalidArgument(f"Got non-callable func={func!r}") + raise InvalidArgument(f"Got non-callable {func=}") except_ = _check_except(except_) _check_style(style) check_type(bool, associative, "associative") check_type(bool, commutative, "commutative") if distributes_over is not None and not callable(distributes_over): raise InvalidArgument( - f"distributes_over={distributes_over!r} must be an operation which " + f"{distributes_over=} must be an operation which " f"distributes over {func.__name__}" ) if not any([associative, commutative, identity, distributes_over]): @@ -1787,7 +1814,7 @@ def ufunc( hypothesis write numpy.matmul """ if not _is_probably_ufunc(func): - raise InvalidArgument(f"func={func!r} does not seem to be a ufunc") + raise InvalidArgument(f"{func=} does not seem to be a ufunc") except_ = _check_except(except_) _check_style(style) diff --git a/contrib/python/hypothesis/py3/hypothesis/extra/numpy.py b/contrib/python/hypothesis/py3/hypothesis/extra/numpy.py index 90dbcd1580..f7f27a7413 100644 --- a/contrib/python/hypothesis/py3/hypothesis/extra/numpy.py +++ b/contrib/python/hypothesis/py3/hypothesis/extra/numpy.py @@ -13,6 +13,7 @@ import math from typing import ( TYPE_CHECKING, Any, + Literal, Mapping, Optional, Sequence, @@ -578,7 +579,9 @@ def dtype_factory(kind, sizes, valid_sizes, endianness): @defines_dtype_strategy def unsigned_integer_dtypes( - *, endianness: str = "?", sizes: Sequence[int] = (8, 16, 32, 64) + *, + endianness: str = "?", + sizes: Sequence[Literal[8, 16, 32, 64]] = (8, 16, 32, 64), ) -> st.SearchStrategy[np.dtype]: """Return a strategy for unsigned integer dtypes. @@ -594,7 +597,9 @@ def unsigned_integer_dtypes( @defines_dtype_strategy def integer_dtypes( - *, endianness: str = "?", sizes: Sequence[int] = (8, 16, 32, 64) + *, + endianness: str = "?", + sizes: Sequence[Literal[8, 16, 32, 64]] = (8, 16, 32, 64), ) -> st.SearchStrategy[np.dtype]: """Return a strategy for signed integer dtypes. @@ -606,7 +611,9 @@ def integer_dtypes( @defines_dtype_strategy def floating_dtypes( - *, endianness: str = "?", sizes: Sequence[int] = (16, 32, 64) + *, + endianness: str = "?", + sizes: Sequence[Literal[16, 32, 64, 96, 128]] = (16, 32, 64), ) -> st.SearchStrategy[np.dtype]: """Return a strategy for floating-point dtypes. @@ -622,7 +629,9 @@ def floating_dtypes( @defines_dtype_strategy def complex_number_dtypes( - *, endianness: str = "?", sizes: Sequence[int] = (64, 128) + *, + endianness: str = "?", + sizes: Sequence[Literal[64, 128, 192, 256]] = (64, 128), ) -> st.SearchStrategy[np.dtype]: """Return a strategy for complex-number dtypes. @@ -885,7 +894,7 @@ def basic_indices( check_type(tuple, shape, "shape") check_argument( all(isinstance(x, int) and x >= 0 for x in shape), - f"shape={shape!r}, but all dimensions must be non-negative integers.", + f"{shape=}, but all dimensions must be non-negative integers.", ) check_type(bool, allow_ellipsis, "allow_ellipsis") check_type(bool, allow_newaxis, "allow_newaxis") @@ -934,7 +943,7 @@ def integer_array_indices( shape: Shape, *, result_shape: st.SearchStrategy[Shape] = array_shapes(), - dtype: D = np.int_, + dtype: D = np.dtype(int), ) -> "st.SearchStrategy[Tuple[NDArray[D], ...]]": """Return a search strategy for tuples of integer-arrays that, when used to index into an array of shape ``shape``, given an array whose shape @@ -978,11 +987,11 @@ def integer_array_indices( check_type(tuple, shape, "shape") check_argument( shape and all(isinstance(x, int) and x > 0 for x in shape), - f"shape={shape!r} must be a non-empty tuple of integers > 0", + f"{shape=} must be a non-empty tuple of integers > 0", ) check_strategy(result_shape, "result_shape") check_argument( - np.issubdtype(dtype, np.integer), f"dtype={dtype!r} must be an integer dtype" + np.issubdtype(dtype, np.integer), f"{dtype=} must be an integer dtype" ) signed = np.issubdtype(dtype, np.signedinteger) diff --git a/contrib/python/hypothesis/py3/hypothesis/extra/pandas/impl.py b/contrib/python/hypothesis/py3/hypothesis/extra/pandas/impl.py index 75af899a97..4801b1ad46 100644 --- a/contrib/python/hypothesis/py3/hypothesis/extra/pandas/impl.py +++ b/contrib/python/hypothesis/py3/hypothesis/extra/pandas/impl.py @@ -35,16 +35,6 @@ from hypothesis.strategies._internal.strategies import Ex, check_strategy from hypothesis.strategies._internal.utils import cacheable, defines_strategy try: - from pandas.api.types import is_categorical_dtype -except ImportError: - - def is_categorical_dtype(dt): - if isinstance(dt, np.dtype): - return False - return dt == "category" - - -try: from pandas.core.arrays.integer import IntegerDtype except ImportError: IntegerDtype = () @@ -79,20 +69,20 @@ def elements_and_dtype(elements, dtype, source=None): f"At least one of {prefix}elements or {prefix}dtype must be provided." ) - with check("is_categorical_dtype"): - if is_categorical_dtype(dtype): + with check("isinstance(dtype, CategoricalDtype)"): + if pandas.api.types.CategoricalDtype.is_dtype(dtype): raise InvalidArgument( f"{prefix}dtype is categorical, which is currently unsupported" ) if isinstance(dtype, type) and issubclass(dtype, IntegerDtype): raise InvalidArgument( - f"Passed dtype={dtype!r} is a dtype class, please pass in an instance of this class." + f"Passed {dtype=} is a dtype class, please pass in an instance of this class." "Otherwise it would be treated as dtype=object" ) if isinstance(dtype, type) and np.dtype(dtype).kind == "O" and dtype is not object: - err_msg = f"Passed dtype={dtype!r} is not a valid Pandas dtype." + err_msg = f"Passed {dtype=} is not a valid Pandas dtype." if issubclass(dtype, datetime): err_msg += ' To generate valid datetimes, pass `dtype="datetime64[ns]"`' raise InvalidArgument(err_msg) @@ -104,11 +94,12 @@ def elements_and_dtype(elements, dtype, source=None): "dtype=object for now, but this will be an error in a future version.", since="2021-12-31", has_codemod=False, + stacklevel=1, ) if isinstance(dtype, st.SearchStrategy): raise InvalidArgument( - f"Passed dtype={dtype!r} is a strategy, but we require a concrete dtype " + f"Passed {dtype=} is a strategy, but we require a concrete dtype " "here. See https://stackoverflow.com/q/74355937 for workaround patterns." ) @@ -653,7 +644,7 @@ def data_frames( value, (float, int, str, bool, datetime, timedelta) ): raise ValueError( - f"Failed to add value={value!r} to column " + f"Failed to add {value=} to column " f"{c.name} with dtype=None. Maybe passing " "dtype=object would help?" ) from err diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/cache.py b/contrib/python/hypothesis/py3/hypothesis/internal/cache.py index 891b2111f5..86241d3737 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/cache.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/cache.py @@ -162,7 +162,7 @@ class GenericCache: Returns the score to associate with the key. """ - raise NotImplementedError() + raise NotImplementedError def on_access(self, key, value, score): """Called every time a key that is already in the map is read or diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/charmap.py b/contrib/python/hypothesis/py3/hypothesis/internal/charmap.py index 80e94e2846..6f41fbf021 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/charmap.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/charmap.py @@ -8,24 +8,27 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. +import codecs import gzip import json import os import sys import tempfile import unicodedata +from functools import lru_cache from typing import Dict, Tuple -from hypothesis.configuration import mkdir_p, storage_directory +from hypothesis.configuration import storage_directory from hypothesis.errors import InvalidArgument +from hypothesis.internal.intervalsets import IntervalSet intervals = Tuple[Tuple[int, int], ...] -cache_type = Dict[Tuple[Tuple[str, ...], int, int, intervals], intervals] +cache_type = Dict[Tuple[Tuple[str, ...], int, int, intervals], IntervalSet] -def charmap_file(): +def charmap_file(fname="charmap"): return storage_directory( - "unicode_data", unicodedata.unidata_version, "charmap.json.gz" + "unicode_data", unicodedata.unidata_version, f"{fname}.json.gz" ) @@ -67,7 +70,7 @@ def charmap(): try: # Write the Unicode table atomically tmpdir = storage_directory("tmp") - mkdir_p(tmpdir) + tmpdir.mkdir(exist_ok=True, parents=True) fd, tmpfile = tempfile.mkstemp(dir=tmpdir) os.close(fd) # Explicitly set the mtime to get reproducible output @@ -95,6 +98,43 @@ def charmap(): return _charmap +@lru_cache(maxsize=None) +def intervals_from_codec(codec_name: str) -> IntervalSet: # pragma: no cover + """Return an IntervalSet of characters which are part of this codec.""" + assert codec_name == codecs.lookup(codec_name).name + fname = charmap_file(f"codec-{codec_name}") + try: + with gzip.GzipFile(fname) as gzf: + encodable_intervals = json.load(gzf) + + except Exception: + # This loop is kinda slow, but hopefully we don't need to do it very often! + encodable_intervals = [] + for i in range(sys.maxunicode + 1): + try: + chr(i).encode(codec_name) + except Exception: # usually _but not always_ UnicodeEncodeError + pass + else: + encodable_intervals.append((i, i)) + + res = IntervalSet(encodable_intervals) + res = res.union(res) + try: + # Write the Unicode table atomically + tmpdir = storage_directory("tmp") + tmpdir.mkdir(exist_ok=True, parents=True) + fd, tmpfile = tempfile.mkstemp(dir=tmpdir) + os.close(fd) + # Explicitly set the mtime to get reproducible output + with gzip.GzipFile(tmpfile, "wb", mtime=1) as o: + o.write(json.dumps(res.intervals).encode()) + os.renames(tmpfile, fname) + except Exception: + pass + return res + + _categories = None @@ -146,130 +186,10 @@ def as_general_categories(cats, name="cats"): return tuple(c for c in cs if c in out) -def _union_intervals(x, y): - """Merge two sequences of intervals into a single tuple of intervals. - - Any integer bounded by `x` or `y` is also bounded by the result. - - >>> _union_intervals([(3, 10)], [(1, 2), (5, 17)]) - ((1, 17),) - """ - if not x: - return tuple((u, v) for u, v in y) - if not y: - return tuple((u, v) for u, v in x) - intervals = sorted(x + y, reverse=True) - result = [intervals.pop()] - while intervals: - # 1. intervals is in descending order - # 2. pop() takes from the RHS. - # 3. (a, b) was popped 1st, then (u, v) was popped 2nd - # 4. Therefore: a <= u - # 5. We assume that u <= v and a <= b - # 6. So we need to handle 2 cases of overlap, and one disjoint case - # | u--v | u----v | u--v | - # | a----b | a--b | a--b | - u, v = intervals.pop() - a, b = result[-1] - if u <= b + 1: - # Overlap cases - result[-1] = (a, max(v, b)) - else: - # Disjoint case - result.append((u, v)) - return tuple(result) - - -def _subtract_intervals(x, y): - """Set difference for lists of intervals. That is, returns a list of - intervals that bounds all values bounded by x that are not also bounded by - y. x and y are expected to be in sorted order. - - For example _subtract_intervals([(1, 10)], [(2, 3), (9, 15)]) would - return [(1, 1), (4, 8)], removing the values 2, 3, 9 and 10 from the - interval. - """ - if not y: - return tuple(x) - x = list(map(list, x)) - i = 0 - j = 0 - result = [] - while i < len(x) and j < len(y): - # Iterate in parallel over x and y. j stays pointing at the smallest - # interval in the left hand side that could still overlap with some - # element of x at index >= i. - # Similarly, i is not incremented until we know that it does not - # overlap with any element of y at index >= j. - - xl, xr = x[i] - assert xl <= xr - yl, yr = y[j] - assert yl <= yr - - if yr < xl: - # The interval at y[j] is strictly to the left of the interval at - # x[i], so will not overlap with it or any later interval of x. - j += 1 - elif yl > xr: - # The interval at y[j] is strictly to the right of the interval at - # x[i], so all of x[i] goes into the result as no further intervals - # in y will intersect it. - result.append(x[i]) - i += 1 - elif yl <= xl: - if yr >= xr: - # x[i] is contained entirely in y[j], so we just skip over it - # without adding it to the result. - i += 1 - else: - # The beginning of x[i] is contained in y[j], so we update the - # left endpoint of x[i] to remove this, and increment j as we - # now have moved past it. Note that this is not added to the - # result as is, as more intervals from y may intersect it so it - # may need updating further. - x[i][0] = yr + 1 - j += 1 - else: - # yl > xl, so the left hand part of x[i] is not contained in y[j], - # so there are some values we should add to the result. - result.append((xl, yl - 1)) - - if yr + 1 <= xr: - # If y[j] finishes before x[i] does, there may be some values - # in x[i] left that should go in the result (or they may be - # removed by a later interval in y), so we update x[i] to - # reflect that and increment j because it no longer overlaps - # with any remaining element of x. - x[i][0] = yr + 1 - j += 1 - else: - # Every element of x[i] other than the initial part we have - # already added is contained in y[j], so we move to the next - # interval. - i += 1 - # Any remaining intervals in x do not overlap with any of y, as if they did - # we would not have incremented j to the end, so can be added to the result - # as they are. - result.extend(x[i:]) - return tuple(map(tuple, result)) - - -def _intervals(s): - """Return a tuple of intervals, covering the codepoints of characters in - `s`. - - >>> _intervals('abcdef0123456789') - ((48, 57), (97, 102)) - """ - intervals = tuple((ord(c), ord(c)) for c in sorted(s)) - return _union_intervals(intervals, intervals) - - category_index_cache = {(): ()} -def _category_key(exclude, include): +def _category_key(cats): """Return a normalised tuple of all Unicode categories that are in `include`, but not in `exclude`. @@ -280,15 +200,9 @@ def _category_key(exclude, include): ('Me', 'Lu', 'Cs') """ cs = categories() - if include is None: - include = set(cs) - else: - include = set(include) - exclude = set(exclude or ()) - assert include.issubset(cs) - assert exclude.issubset(cs) - include -= exclude - return tuple(c for c in cs if c in include) + if cats is None: + cats = set(cs) + return tuple(c for c in cs if c in cats) def _query_for_key(key): @@ -306,36 +220,37 @@ def _query_for_key(key): pass assert key if set(key) == set(categories()): - result = ((0, sys.maxunicode),) + result = IntervalSet([(0, sys.maxunicode)]) else: - result = _union_intervals(_query_for_key(key[:-1]), charmap()[key[-1]]) - category_index_cache[key] = result - return result + result = IntervalSet(_query_for_key(key[:-1])).union( + IntervalSet(charmap()[key[-1]]) + ) + assert isinstance(result, IntervalSet) + category_index_cache[key] = result.intervals + return result.intervals limited_category_index_cache: cache_type = {} def query( - exclude_categories=(), - include_categories=None, + *, + categories=None, min_codepoint=None, max_codepoint=None, include_characters="", exclude_characters="", ): """Return a tuple of intervals covering the codepoints for all characters - that meet the criteria (min_codepoint <= codepoint(c) <= max_codepoint and - any(cat in include_categories for cat in categories(c)) and all(cat not in - exclude_categories for cat in categories(c)) or (c in include_characters) + that meet the criteria. >>> query() ((0, 1114111),) >>> query(min_codepoint=0, max_codepoint=128) ((0, 128),) - >>> query(min_codepoint=0, max_codepoint=128, include_categories=['Lu']) + >>> query(min_codepoint=0, max_codepoint=128, categories=['Lu']) ((65, 90),) - >>> query(min_codepoint=0, max_codepoint=128, include_categories=['Lu'], + >>> query(min_codepoint=0, max_codepoint=128, categories=['Lu'], ... include_characters='☃') ((65, 90), (9731, 9731)) """ @@ -343,15 +258,15 @@ def query( min_codepoint = 0 if max_codepoint is None: max_codepoint = sys.maxunicode - catkey = _category_key(exclude_categories, include_categories) - character_intervals = _intervals(include_characters or "") - exclude_intervals = _intervals(exclude_characters or "") + catkey = _category_key(categories) + character_intervals = IntervalSet.from_string(include_characters or "") + exclude_intervals = IntervalSet.from_string(exclude_characters or "") qkey = ( catkey, min_codepoint, max_codepoint, - character_intervals, - exclude_intervals, + character_intervals.intervals, + exclude_intervals.intervals, ) try: return limited_category_index_cache[qkey] @@ -362,8 +277,6 @@ def query( for u, v in base: if v >= min_codepoint and u <= max_codepoint: result.append((max(u, min_codepoint), min(v, max_codepoint))) - result = tuple(result) - result = _union_intervals(result, character_intervals) - result = _subtract_intervals(result, exclude_intervals) + result = (IntervalSet(result) | character_intervals) - exclude_intervals limited_category_index_cache[qkey] = result return result diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/compat.py b/contrib/python/hypothesis/py3/hypothesis/internal/compat.py index 519feb6588..3eaed1eba1 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/compat.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/compat.py @@ -14,62 +14,7 @@ import platform import sys import typing from functools import partial -from typing import Any, ForwardRef, Tuple - -try: - from typing import get_args as get_args -except ImportError: - # remove at Python 3.7 end-of-life - from collections.abc import Callable as _Callable - - def get_args( - tp: Any, - ) -> Tuple[Any, ...]: # pragma: no cover - """ - Examples - -------- - >>> assert get_args(int) == () - >>> assert get_args(Dict[str, int]) == (str, int) - >>> assert get_args(Union[int, Union[T, int], str][int]) == (int, str) - >>> assert get_args(Union[int, Tuple[T, int]][str]) == (int, Tuple[str, int]) - >>> assert get_args(Callable[[], T][int]) == ([], int) - """ - if hasattr(tp, "__origin__") and hasattr(tp, "__args__"): - args = tp.__args__ - if ( - getattr(tp, "__origin__", None) is _Callable - and args - and args[0] is not Ellipsis - ): - args = (list(args[:-1]), args[-1]) - return args - return () - - -try: - from typing import get_origin as get_origin -except ImportError: - # remove at Python 3.7 end-of-life - from collections.abc import Callable as _Callable # noqa: F811 - - def get_origin(tp: Any) -> typing.Optional[Any]: # type: ignore # pragma: no cover - """Get the unsubscripted version of a type. - This supports generic types, Callable, Tuple, Union, Literal, Final and ClassVar. - Return None for unsupported types. Examples:: - get_origin(Literal[42]) is Literal - get_origin(int) is None - get_origin(ClassVar[int]) is ClassVar - get_origin(Generic) is Generic - get_origin(Generic[T]) is Generic - get_origin(Union[T, int]) is Union - get_origin(List[Tuple[T, T]][int]) == list - """ - if hasattr(tp, "__origin__"): - return tp.__origin__ - if tp is typing.Generic: - return typing.Generic - return None - +from typing import Any, ForwardRef, get_args try: BaseExceptionGroup = BaseExceptionGroup @@ -98,6 +43,15 @@ GRAALPY = platform.python_implementation() == "GraalVM" WINDOWS = platform.system() == "Windows" +def add_note(exc, note): + try: + exc.add_note(note) + except AttributeError: + if not hasattr(exc, "__notes__"): + exc.__notes__ = [] + exc.__notes__.append(note) + + def escape_unicode_characters(s: str) -> str: return codecs.encode(s, "unicode_escape").decode("ascii") @@ -130,7 +84,7 @@ def is_typed_named_tuple(cls): def _hint_and_args(x): - return (x,) + get_args(x) + return (x, *get_args(x)) def get_type_hints(thing): @@ -204,49 +158,6 @@ def get_type_hints(thing): return hints -def update_code_location(code, newfile, newlineno): - """Take a code object and lie shamelessly about where it comes from. - - Why do we want to do this? It's for really shallow reasons involving - hiding the hypothesis_temporary_module code from test runners like - pytest's verbose mode. This is a vastly disproportionate terrible - hack that I've done purely for vanity, and if you're reading this - code you're probably here because it's broken something and now - you're angry at me. Sorry. - """ - if hasattr(code, "replace"): - # Python 3.8 added positional-only params (PEP 570), and thus changed - # the layout of code objects. In beta1, the `.replace()` method was - # added to facilitate future-proof code. See BPO-37032 for details. - return code.replace(co_filename=newfile, co_firstlineno=newlineno) - - else: # pragma: no cover - # This field order is accurate for 3.5 - 3.7, but not 3.8 when a new field - # was added for positional-only arguments. However it also added a .replace() - # method that we use instead of field indices, so they're fine as-is. - CODE_FIELD_ORDER = [ - "co_argcount", - "co_kwonlyargcount", - "co_nlocals", - "co_stacksize", - "co_flags", - "co_code", - "co_consts", - "co_names", - "co_varnames", - "co_filename", - "co_name", - "co_firstlineno", - "co_lnotab", - "co_freevars", - "co_cellvars", - ] - unpacked = [getattr(code, name) for name in CODE_FIELD_ORDER] - unpacked[CODE_FIELD_ORDER.index("co_filename")] = newfile - unpacked[CODE_FIELD_ORDER.index("co_firstlineno")] = newlineno - return type(code)(*unpacked) - - # Under Python 2, math.floor and math.ceil returned floats, which cannot # represent large integers - eg `float(2**53) == float(2**53 + 1)`. # We therefore implement them entirely in (long) integer operations. diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/choicetree.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/choicetree.py index 38bcc3a571..c5de31ab3a 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/choicetree.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/choicetree.py @@ -89,7 +89,7 @@ class Chooser: node.children[i] = DeadNode node.live_child_count -= 1 assert node.live_child_count == 0 - raise DeadBranch() + raise DeadBranch def finish(self) -> Sequence[int]: """Record the decisions made in the underlying tree and return diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/data.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/data.py index 0b43d9c1b1..c1337cde0f 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/data.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/data.py @@ -240,7 +240,7 @@ class ExampleProperty: self.bytes_read = blocks.endpoints[self.block_count] self.block(self.block_count) self.block_count += 1 - self.__pop(False) + self.__pop(discarded=False) elif record >= START_EXAMPLE_RECORD: self.__push(record - START_EXAMPLE_RECORD) else: @@ -248,19 +248,19 @@ class ExampleProperty: STOP_EXAMPLE_DISCARD_RECORD, STOP_EXAMPLE_NO_DISCARD_RECORD, ) - self.__pop(record == STOP_EXAMPLE_DISCARD_RECORD) + self.__pop(discarded=record == STOP_EXAMPLE_DISCARD_RECORD) return self.finish() def __push(self, label_index: int) -> None: i = self.example_count assert i < len(self.examples) - self.start_example(i, label_index) + self.start_example(i, label_index=label_index) self.example_count += 1 self.example_stack.append(i) - def __pop(self, discarded: bool) -> None: + def __pop(self, *, discarded: bool) -> None: i = self.example_stack.pop() - self.stop_example(i, discarded) + self.stop_example(i, discarded=discarded) def begin(self) -> None: """Called at the beginning of the run to initialise any @@ -276,7 +276,7 @@ class ExampleProperty: """Called with each ``draw_bits`` call, with ``i`` the index of the corresponding block in ``self.examples.blocks``""" - def stop_example(self, i: int, discarded: bool) -> None: + def stop_example(self, i: int, *, discarded: bool) -> None: """Called at the end of each example, with ``i`` the index of the example and ``discarded`` being ``True`` if ``stop_example`` was called with ``discard=True``.""" @@ -342,7 +342,7 @@ class ExampleRecord: self.labels.append(label) self.trail.append(START_EXAMPLE_RECORD + i) - def stop_example(self, discard: bool) -> None: + def stop_example(self, *, discard: bool) -> None: if discard: self.trail.append(STOP_EXAMPLE_DISCARD_RECORD) else: @@ -382,7 +382,7 @@ class Examples: def start_example(self, i: int, label_index: int) -> None: self.starts[i] = self.bytes_read - def stop_example(self, i: int, discarded: bool) -> None: + def stop_example(self, i: int, *, discarded: bool) -> None: self.ends[i] = self.bytes_read def finish(self) -> Tuple[IntList, IntList]: @@ -407,7 +407,7 @@ class Examples: def finish(self) -> FrozenSet[int]: return frozenset(self.result) - def stop_example(self, i: int, discarded: bool) -> None: + def stop_example(self, i: int, *, discarded: bool) -> None: if discarded: self.result.add(i) @@ -422,7 +422,7 @@ class Examples: if not self.examples.blocks.trivial(i): self.nontrivial[self.example_stack[-1]] = 1 - def stop_example(self, i: int, discarded: bool) -> None: + def stop_example(self, i: int, *, discarded: bool) -> None: if self.nontrivial[i]: if self.example_stack: self.nontrivial[self.example_stack[-1]] = 1 @@ -435,7 +435,7 @@ class Examples: trivial: FrozenSet[int] = calculated_example_property(_trivial) class _parentage(ExampleProperty): - def stop_example(self, i: int, discarded: bool) -> None: + def stop_example(self, i: int, *, discarded: bool) -> None: if i > 0: self.result[i] = self.example_stack[-1] @@ -746,7 +746,7 @@ class DataObserver: Note that this is called after ``freeze`` has completed. """ - def draw_bits(self, n_bits: int, forced: bool, value: int) -> None: + def draw_bits(self, n_bits: int, *, forced: bool, value: int) -> None: """Called when ``draw_bits`` is called on on the observed ``ConjectureData``. * ``n_bits`` is the number of bits drawn. @@ -973,14 +973,14 @@ class ConjectureData: self.__example_record.start_example(label) self.labels_for_structure_stack.append({label}) - def stop_example(self, discard: bool = False) -> None: + def stop_example(self, *, discard: bool = False) -> None: if self.frozen: return if discard: self.has_discards = True self.depth -= 1 assert self.depth >= -1 - self.__example_record.stop_example(discard) + self.__example_record.stop_example(discard=discard) labels_for_structure = self.labels_for_structure_stack.pop() @@ -1082,7 +1082,7 @@ class ConjectureData: buf = bytes(buf) result = int_from_bytes(buf) - self.observer.draw_bits(n, forced is not None, result) + self.observer.draw_bits(n, forced=forced is not None, value=result) self.__example_record.draw_bits(n, forced) initial = self.index diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/datatree.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/datatree.py index e61f33732a..b39666eec1 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/datatree.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/datatree.py @@ -283,18 +283,18 @@ class DataTree: n_bits, forced=node.values[i] if i in node.forced else None ) if v != previous: - raise PreviouslyUnseenBehaviour() + raise PreviouslyUnseenBehaviour if isinstance(node.transition, Conclusion): t = node.transition data.conclude_test(t.status, t.interesting_origin) elif node.transition is None: - raise PreviouslyUnseenBehaviour() + raise PreviouslyUnseenBehaviour elif isinstance(node.transition, Branch): v = data.draw_bits(node.transition.bit_length) try: node = node.transition.children[v] except KeyError as err: - raise PreviouslyUnseenBehaviour() from err + raise PreviouslyUnseenBehaviour from err else: assert isinstance(node.transition, Killed) data.observer.kill_branch() diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/dfa/__init__.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/dfa/__init__.py index 40b87f69b6..8beb4ad588 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/dfa/__init__.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/dfa/__init__.py @@ -53,16 +53,16 @@ class DFA: @property def start(self): """Returns the starting state.""" - raise NotImplementedError() + raise NotImplementedError def is_accepting(self, i): """Returns if state ``i`` is an accepting one.""" - raise NotImplementedError() + raise NotImplementedError def transition(self, i, c): """Returns the state that i transitions to on reading character c from a string.""" - raise NotImplementedError() + raise NotImplementedError @property def alphabet(self): diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/engine.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/engine.py index 9e6e19a4e0..6c5fc76cc8 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/engine.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/engine.py @@ -9,7 +9,6 @@ # obtain one at https://mozilla.org/MPL/2.0/. import math -import sys import time from collections import defaultdict from contextlib import contextmanager @@ -37,7 +36,7 @@ from hypothesis.internal.conjecture.datatree import ( PreviouslyUnseenBehaviour, TreeRecordingObserver, ) -from hypothesis.internal.conjecture.junkdrawer import clamp, stack_depth_of_caller +from hypothesis.internal.conjecture.junkdrawer import clamp, ensure_free_stackframes from hypothesis.internal.conjecture.pareto import NO_SCORE, ParetoFront, ParetoOptimiser from hypothesis.internal.conjecture.shrinker import Shrinker, sort_key from hypothesis.internal.healthcheck import fail_health_check @@ -81,6 +80,7 @@ class ConjectureRunner: def __init__( self, test_function, + *, settings=None, random=None, database_key=None, @@ -133,9 +133,6 @@ class ConjectureRunner: # executed test case. self.__data_cache = LRUReusedCache(CACHE_SIZE) - # We ensure that the test has this much stack space remaining, no matter - # the size of the stack when called, to de-flake RecursionErrors (#2494). - self.__recursion_limit = sys.getrecursionlimit() self.__pending_call_explanation = None def explain_next_call_as(self, explanation): @@ -169,32 +166,22 @@ class ConjectureRunner: """Run ``self._test_function``, but convert a ``StopTest`` exception into a normal return and avoid raising Flaky for RecursionErrors. """ - depth = stack_depth_of_caller() - # Because we add to the recursion limit, to be good citizens we also add - # a check for unbounded recursion. The default limit is 1000, so this can - # only ever trigger if something really strange is happening and it's hard - # to imagine an intentionally-deeply-recursive use of this code. - assert depth <= 1000, ( - "Hypothesis would usually add %d to the stack depth of %d here, " - "but we are already much deeper than expected. Aborting now, to " - "avoid extending the stack limit in an infinite loop..." - % (self.__recursion_limit, depth) - ) - try: - sys.setrecursionlimit(depth + self.__recursion_limit) - self._test_function(data) - except StopTest as e: - if e.testcounter == data.testcounter: - # This StopTest has successfully stopped its test, and can now - # be discarded. - pass - else: - # This StopTest was raised by a different ConjectureData. We - # need to re-raise it so that it will eventually reach the - # correct engine. - raise - finally: - sys.setrecursionlimit(self.__recursion_limit) + # We ensure that the test has this much stack space remaining, no + # matter the size of the stack when called, to de-flake RecursionErrors + # (#2494, #3671). + with ensure_free_stackframes(): + try: + self._test_function(data) + except StopTest as e: + if e.testcounter == data.testcounter: + # This StopTest has successfully stopped its test, and can now + # be discarded. + pass + else: + # This StopTest was raised by a different ConjectureData. We + # need to re-raise it so that it will eventually reach the + # correct engine. + raise def test_function(self, data): if self.__pending_call_explanation is not None: @@ -563,7 +550,7 @@ class ConjectureRunner: self.statistics["targets"] = dict(self.best_observed_targets) self.debug(f"exit_with({reason.name})") self.exit_reason = reason - raise RunIsComplete() + raise RunIsComplete def should_generate_more(self): # End the generation phase where we would have ended it if no bugs had @@ -993,7 +980,7 @@ class ConjectureRunner: explain=Phase.explain in self.settings.phases, ) - def cached_test_function(self, buffer, error_on_discard=False, extend=0): + def cached_test_function(self, buffer, *, error_on_discard=False, extend=0): """Checks the tree to see if we've tested this buffer, and returns the previous result if we have. @@ -1028,7 +1015,7 @@ class ConjectureRunner: class DiscardObserver(DataObserver): def kill_branch(self): - raise ContainsDiscard() + raise ContainsDiscard observer = DiscardObserver() else: diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/junkdrawer.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/junkdrawer.py index dc3d0a0f9c..ec12b028b8 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/junkdrawer.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/junkdrawer.py @@ -14,6 +14,7 @@ anything that lives here, please move it.""" import array import sys +import warnings from random import Random from typing import ( Callable, @@ -30,6 +31,8 @@ from typing import ( overload, ) +from hypothesis.errors import HypothesisWarning + ARRAY_CODES = ["B", "H", "I", "L", "Q", "O"] @@ -271,6 +274,42 @@ def stack_depth_of_caller() -> int: return size +class ensure_free_stackframes: + """Context manager that ensures there are at least N free stackframes (for + a reasonable value of N). + """ + + def __enter__(self): + cur_depth = stack_depth_of_caller() + self.old_maxdepth = sys.getrecursionlimit() + # The default CPython recursionlimit is 1000, but pytest seems to bump + # it to 3000 during test execution. Let's make it something reasonable: + self.new_maxdepth = cur_depth + 2000 + # Because we add to the recursion limit, to be good citizens we also + # add a check for unbounded recursion. The default limit is typically + # 1000/3000, so this can only ever trigger if something really strange + # is happening and it's hard to imagine an + # intentionally-deeply-recursive use of this code. + assert cur_depth <= 1000, ( + "Hypothesis would usually add %d to the stack depth of %d here, " + "but we are already much deeper than expected. Aborting now, to " + "avoid extending the stack limit in an infinite loop..." + % (self.new_maxdepth - self.old_maxdepth, self.old_maxdepth) + ) + sys.setrecursionlimit(self.new_maxdepth) + + def __exit__(self, *args, **kwargs): + if self.new_maxdepth == sys.getrecursionlimit(): + sys.setrecursionlimit(self.old_maxdepth) + else: # pragma: no cover + warnings.warn( + "The recursion limit will not be reset, since it was changed " + "from another thread or during execution of a test.", + HypothesisWarning, + stacklevel=2, + ) + + def find_integer(f: Callable[[int], bool]) -> int: """Finds a (hopefully large) integer such that f(n) is True and f(n + 1) is False. diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinker.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinker.py index 4503804afd..82ff4ab88d 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinker.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinker.py @@ -244,7 +244,7 @@ class Shrinker: """ - def derived_value(fn): # noqa: B902 + def derived_value(fn): """It's useful during shrinking to have access to derived values of the current shrink target. @@ -308,7 +308,7 @@ class Shrinker: def cached(self, *keys): def accept(f): - cache_key = (f.__name__,) + keys + cache_key = (f.__name__, *keys) try: return self.cached_calculations[cache_key] except KeyError: @@ -412,7 +412,7 @@ class Shrinker: result = self.engine.cached_test_function(buffer) self.incorporate_test_data(result) if self.calls - self.calls_at_last_shrink >= self.max_stall: - raise StopShrinking() + raise StopShrinking return result def debug(self, msg): @@ -547,7 +547,7 @@ class Shrinker: # Turns out this was a variable-length part, so grab the infix... if result.status == Status.OVERRUN: - continue # pragma: no cover + continue # pragma: no cover # flakily covered if not ( len(buf_attempt_fixed) == len(result.buffer) and result.buffer.endswith(buffer[end:]) @@ -568,12 +568,8 @@ class Shrinker: chunks[(start, end)].append(result.buffer[start:res_end]) result = self.engine.cached_test_function(buf_attempt_fixed) - if ( - result.status == Status.OVERRUN - or len(buf_attempt_fixed) != len(result.buffer) - or not result.buffer.endswith(buffer[end:]) - ): - raise NotImplementedError("This should never happen") + if result.status == Status.OVERRUN: + continue # pragma: no cover # flakily covered else: chunks[(start, end)].append(result.buffer[start:end]) @@ -1553,7 +1549,7 @@ class ShrinkPass: shrinks = attr.ib(default=0) deletions = attr.ib(default=0) - def step(self, random_order=False): + def step(self, *, random_order=False): tree = self.shrinker.shrink_pass_choice_trees[self] if tree.exhausted: return False diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/common.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/common.py index 24df7b90ab..090324f839 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/common.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/common.py @@ -17,7 +17,15 @@ class Shrinker: and simpler.""" def __init__( - self, initial, predicate, random, full=False, debug=False, name=None, **kwargs + self, + initial, + predicate, + random, + *, + full=False, + debug=False, + name=None, + **kwargs, ): self.setup(**kwargs) self.current = self.make_immutable(initial) @@ -110,9 +118,7 @@ class Shrinker: self.check_invariants(value) if not self.left_is_better(value, self.current): if value != self.current and (value == value): - self.debug( - f"Rejected {value!r} as worse than self.current={self.current!r}" - ) + self.debug(f"Rejected {value!r} as worse than {self.current=}") return False if value in self.__seen: return False @@ -148,7 +154,7 @@ class Shrinker: Does nothing by default. """ - raise NotImplementedError() + raise NotImplementedError def short_circuit(self): """Possibly attempt to do some shrinking. @@ -156,14 +162,14 @@ class Shrinker: If this returns True, the ``run`` method will terminate early without doing any more work. """ - raise NotImplementedError() + raise NotImplementedError def left_is_better(self, left, right): """Returns True if the left is strictly simpler than the right according to the standards of this shrinker.""" - raise NotImplementedError() + raise NotImplementedError def run_step(self): """Run a single step of the main shrink loop, attempting to improve the current value.""" - raise NotImplementedError() + raise NotImplementedError diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/dfas.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/dfas.py index 81ea67a155..050672c5da 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/dfas.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/dfas.py @@ -8,9 +8,18 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. +""" +This is a module for learning new DFAs that help normalize test +functions. That is, given a test function that sometimes shrinks +to one thing and sometimes another, this module is designed to +help learn new DFA-based shrink passes that will cause it to +always shrink to the same thing. +""" + import hashlib import math from itertools import islice +from pathlib import Path from hypothesis import HealthCheck, settings from hypothesis.errors import HypothesisException @@ -18,16 +27,10 @@ from hypothesis.internal.conjecture.data import ConjectureResult, Status from hypothesis.internal.conjecture.dfa.lstar import LStar from hypothesis.internal.conjecture.shrinking.learned_dfas import ( SHRINKING_DFAS, - __file__ as learned_dfa_file, + __file__ as _learned_dfa_file, ) -""" -This is a module for learning new DFAs that help normalize test -functions. That is, given a test function that sometimes shrinks -to one thing and sometimes another, this module is designed to -help learn new DFA-based shrink passes that will cause it to -always shrink to the same thing. -""" +learned_dfa_file = Path(_learned_dfa_file) class FailedToNormalise(HypothesisException): @@ -38,8 +41,7 @@ def update_learned_dfas(): """Write any modifications to the SHRINKING_DFAS dictionary back to the learned DFAs file.""" - with open(learned_dfa_file) as i: - source = i.read() + source = learned_dfa_file.read_text(encoding="utf-8") lines = source.splitlines() @@ -60,8 +62,7 @@ def update_learned_dfas(): new_source = "\n".join(lines) + "\n" if new_source != source: - with open(learned_dfa_file, "w") as o: - o.write(new_source) + learned_dfa_file.write_text(new_source, encoding="utf-8") def learn_a_new_dfa(runner, u, v, predicate): diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/floats.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/floats.py index ceab3f5f0f..f05cdfff30 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/floats.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/floats.py @@ -41,6 +41,10 @@ class Float(Shrinker): return lex1 < lex2 def short_circuit(self): + # We check for a bunch of standard "large" floats. If we're currently + # worse than them and the shrink downwards doesn't help, abort early + # because there's not much useful we can do here. + for g in [sys.float_info.max, math.inf, math.nan]: self.consider(g) @@ -53,10 +57,6 @@ class Float(Shrinker): return self.current >= MAX_PRECISE_INTEGER def run_step(self): - # We check for a bunch of standard "large" floats. If we're currently - # worse than them and the shrink downwards doesn't help, abort early - # because there's not much useful we can do here. - # Finally we get to the important bit: Each of these is a small change # to the floating point number that corresponds to a large change in # the lexical representation. Trying these ensures that our floating @@ -65,18 +65,26 @@ class Float(Shrinker): # change that would require shifting the exponent while not changing # the float value much. - for g in [math.floor(self.current), math.ceil(self.current)]: - self.consider(g) + # First, try dropping precision bits by rounding the scaled value. We + # try values ordered from least-precise (integer) to more precise, ie. + # approximate lexicographical order. Once we find an acceptable shrink, + # self.consider discards the remaining attempts early and skips test + # invocation. The loop count sets max fractional bits to keep, and is a + # compromise between completeness and performance. + + for p in range(10): + scaled = self.current * 2**p # note: self.current may change in loop + for truncate in [math.floor, math.ceil]: + self.consider(truncate(scaled) / 2**p) if self.consider(int(self.current)): self.debug("Just an integer now") self.delegate(Integer, convert_to=int, convert_from=float) return - m, n = self.current.as_integer_ratio() - i, r = divmod(m, n) - # Now try to minimize the top part of the fraction as an integer. This # basically splits the float as k + x with 0 <= x < 1 and minimizes # k as an integer, but without the precision issues that would have. - self.call_shrinker(Integer, i, lambda k: self.consider((i * n + r) / n)) + m, n = self.current.as_integer_ratio() + i, r = divmod(m, n) + self.call_shrinker(Integer, i, lambda k: self.consider((k * n + r) / n)) diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/lexical.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/lexical.py index c755f456b4..569561c4ed 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/lexical.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/lexical.py @@ -39,12 +39,11 @@ class Lexical(Shrinker): def current_int(self): return int_from_bytes(self.current) - def minimize_as_integer(self, full=False): + def minimize_as_integer(self): Integer.shrink( self.current_int, lambda c: c == self.current_int or self.incorporate_int(c), random=self.random, - full=full, ) def partial_sort(self): diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/utils.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/utils.py index 84589edc4e..7402bb47d3 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/utils.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/utils.py @@ -279,11 +279,7 @@ def biased_coin( # becomes i > falsey. result = i > falsey - if i > 1: # pragma: no branch - # Thanks to bytecode optimisations on CPython >= 3.7 and PyPy - # (see https://bugs.python.org/issue2506), coverage incorrectly - # thinks that this condition is always true. You can trivially - # check by adding `else: assert False` and running the tests. + if i > 1: data.draw_bits(bits, forced=int(result)) break data.stop_example() @@ -471,7 +467,7 @@ class many: SMALLEST_POSITIVE_FLOAT: float = next_up(0.0) or sys.float_info.min -@lru_cache() +@lru_cache def _calc_p_continue(desired_avg: float, max_size: int) -> float: """Return the p_continue which will generate the desired average size.""" assert desired_avg <= max_size, (desired_avg, max_size) diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/coverage.py b/contrib/python/hypothesis/py3/hypothesis/internal/coverage.py index 56a7f4fd9a..40f609b75a 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/coverage.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/coverage.py @@ -61,7 +61,7 @@ if IN_COVERAGE_TESTS: if key in written: return written.add(key) - with open("branch-check", "a") as log: + with open("branch-check", mode="a", encoding="utf-8") as log: log.write(json.dumps({"name": name, "value": value}) + "\n") description_stack = [] diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/entropy.py b/contrib/python/hypothesis/py3/hypothesis/internal/entropy.py index fb0ca2b27e..8715635462 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/entropy.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/entropy.py @@ -22,10 +22,7 @@ from hypothesis.errors import HypothesisWarning, InvalidArgument from hypothesis.internal.compat import GRAALPY, PYPY if TYPE_CHECKING: - if sys.version_info >= (3, 8): - from typing import Protocol - else: - from typing_extensions import Protocol + from typing import Protocol # we can't use this at runtime until from_type supports # protocols -- breaks ghostwriter tests @@ -113,7 +110,7 @@ def register_random(r: RandomLike) -> None: register_random(rng) """ if not (hasattr(r, "seed") and hasattr(r, "getstate") and hasattr(r, "setstate")): - raise InvalidArgument(f"r={r!r} does not have all the required methods") + raise InvalidArgument(f"{r=} does not have all the required methods") if r in RANDOMS_TO_MANAGE.values(): return @@ -127,18 +124,17 @@ def register_random(r: RandomLike) -> None: f"`register_random` was passed `r={r}` which will be " "garbage collected immediately after `register_random` creates a " "weakref to it. This will prevent Hypothesis from managing this " - "source of RNG. See the docs for `register_random` for more " + "PRNG. See the docs for `register_random` for more " "details." ) else: warnings.warn( - HypothesisWarning( - "It looks like `register_random` was passed an object " - "that could be garbage collected immediately after " - "`register_random` creates a weakref to it. This will " - "prevent Hypothesis from managing this source of RNG. " - "See the docs for `register_random` for more details." - ) + "It looks like `register_random` was passed an object that could " + "be garbage collected immediately after `register_random` creates " + "a weakref to it. This will prevent Hypothesis from managing this " + "PRNG. See the docs for `register_random` for more details.", + HypothesisWarning, + stacklevel=2, ) RANDOMS_TO_MANAGE[next(_RKEY)] = r @@ -156,7 +152,8 @@ def get_seeder_and_restorer( to force determinism on simulation or scheduling frameworks which avoid using the global random state. See e.g. #1709. """ - assert isinstance(seed, int) and 0 <= seed < 2**32 + assert isinstance(seed, int) + assert 0 <= seed < 2**32 states: dict = {} if "numpy" in sys.modules: diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/escalation.py b/contrib/python/hypothesis/py3/hypothesis/internal/escalation.py index 64f214a0c5..605ea52e97 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/escalation.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/escalation.py @@ -74,7 +74,7 @@ def escalate_hypothesis_internal_error(): filepath = None if tb is None else traceback.extract_tb(tb)[-1][0] if is_hypothesis_file(filepath) and not isinstance( - e, (HypothesisException,) + HYPOTHESIS_CONTROL_EXCEPTIONS + e, (HypothesisException, *HYPOTHESIS_CONTROL_EXCEPTIONS) ): raise @@ -142,7 +142,7 @@ def _get_exceptioninfo(): with contextlib.suppress(Exception): # From Pytest 7, __init__ warns on direct calls. return sys.modules["pytest"].ExceptionInfo.from_exc_info - if "_pytest._code" in sys.modules: # pragma: no cover # old versions only + if "_pytest._code" in sys.modules: # old versions only with contextlib.suppress(Exception): return sys.modules["_pytest._code"].ExceptionInfo return None # pragma: no cover # coverage tests always use pytest diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/floats.py b/contrib/python/hypothesis/py3/hypothesis/internal/floats.py index d2cb2c94f8..e0cd95d374 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/floats.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/floats.py @@ -119,6 +119,7 @@ assert width_smallest_normals[64] == float_info.min def make_float_clamper( min_float: float = 0.0, max_float: float = math.inf, + *, allow_zero: bool = False, # Allows +0.0 (even if minfloat > 0) ) -> Optional[Callable[[float], float]]: """ diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/intervalsets.py b/contrib/python/hypothesis/py3/hypothesis/internal/intervalsets.py index 5bdd731d2d..a4d8851581 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/intervalsets.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/intervalsets.py @@ -10,6 +10,16 @@ class IntervalSet: + @classmethod + def from_string(cls, s): + """Return a tuple of intervals, covering the codepoints of characters in `s`. + + >>> IntervalSet.from_string('abcdef0123456789') + ((48, 57), (97, 102)) + """ + x = cls((ord(c), ord(c)) for c in sorted(s)) + return x.union(x) + def __init__(self, intervals): self.intervals = tuple(intervals) self.offsets = [0] @@ -49,6 +59,13 @@ class IntervalSet: assert r <= v return r + def __contains__(self, elem): + if isinstance(elem, str): + elem = ord(elem) + assert isinstance(elem, int) + assert 0 <= elem <= 0x10FFFF + return any(start <= elem <= end for start, end in self.intervals) + def __repr__(self): return f"IntervalSet({self.intervals!r})" @@ -69,3 +86,145 @@ class IntervalSet: if value <= v: return offset + (value - u) return self.size + + def __or__(self, other): + return self.union(other) + + def __sub__(self, other): + return self.difference(other) + + def __and__(self, other): + return self.intersection(other) + + def union(self, other): + """Merge two sequences of intervals into a single tuple of intervals. + + Any integer bounded by `x` or `y` is also bounded by the result. + + >>> union([(3, 10)], [(1, 2), (5, 17)]) + ((1, 17),) + """ + assert isinstance(other, type(self)) + x = self.intervals + y = other.intervals + if not x: + return IntervalSet((u, v) for u, v in y) + if not y: + return IntervalSet((u, v) for u, v in x) + intervals = sorted(x + y, reverse=True) + result = [intervals.pop()] + while intervals: + # 1. intervals is in descending order + # 2. pop() takes from the RHS. + # 3. (a, b) was popped 1st, then (u, v) was popped 2nd + # 4. Therefore: a <= u + # 5. We assume that u <= v and a <= b + # 6. So we need to handle 2 cases of overlap, and one disjoint case + # | u--v | u----v | u--v | + # | a----b | a--b | a--b | + u, v = intervals.pop() + a, b = result[-1] + if u <= b + 1: + # Overlap cases + result[-1] = (a, max(v, b)) + else: + # Disjoint case + result.append((u, v)) + return IntervalSet(result) + + def difference(self, other): + """Set difference for lists of intervals. That is, returns a list of + intervals that bounds all values bounded by x that are not also bounded by + y. x and y are expected to be in sorted order. + + For example difference([(1, 10)], [(2, 3), (9, 15)]) would + return [(1, 1), (4, 8)], removing the values 2, 3, 9 and 10 from the + interval. + """ + assert isinstance(other, type(self)) + x = self.intervals + y = other.intervals + if not y: + return IntervalSet(x) + x = list(map(list, x)) + i = 0 + j = 0 + result = [] + while i < len(x) and j < len(y): + # Iterate in parallel over x and y. j stays pointing at the smallest + # interval in the left hand side that could still overlap with some + # element of x at index >= i. + # Similarly, i is not incremented until we know that it does not + # overlap with any element of y at index >= j. + + xl, xr = x[i] + assert xl <= xr + yl, yr = y[j] + assert yl <= yr + + if yr < xl: + # The interval at y[j] is strictly to the left of the interval at + # x[i], so will not overlap with it or any later interval of x. + j += 1 + elif yl > xr: + # The interval at y[j] is strictly to the right of the interval at + # x[i], so all of x[i] goes into the result as no further intervals + # in y will intersect it. + result.append(x[i]) + i += 1 + elif yl <= xl: + if yr >= xr: + # x[i] is contained entirely in y[j], so we just skip over it + # without adding it to the result. + i += 1 + else: + # The beginning of x[i] is contained in y[j], so we update the + # left endpoint of x[i] to remove this, and increment j as we + # now have moved past it. Note that this is not added to the + # result as is, as more intervals from y may intersect it so it + # may need updating further. + x[i][0] = yr + 1 + j += 1 + else: + # yl > xl, so the left hand part of x[i] is not contained in y[j], + # so there are some values we should add to the result. + result.append((xl, yl - 1)) + + if yr + 1 <= xr: + # If y[j] finishes before x[i] does, there may be some values + # in x[i] left that should go in the result (or they may be + # removed by a later interval in y), so we update x[i] to + # reflect that and increment j because it no longer overlaps + # with any remaining element of x. + x[i][0] = yr + 1 + j += 1 + else: + # Every element of x[i] other than the initial part we have + # already added is contained in y[j], so we move to the next + # interval. + i += 1 + # Any remaining intervals in x do not overlap with any of y, as if they did + # we would not have incremented j to the end, so can be added to the result + # as they are. + result.extend(x[i:]) + return IntervalSet(map(tuple, result)) + + def intersection(self, other): + """Set intersection for lists of intervals.""" + assert isinstance(other, type(self)), other + intervals = [] + i = j = 0 + while i < len(self.intervals) and j < len(other.intervals): + u, v = self.intervals[i] + U, V = other.intervals[j] + if u > V: + j += 1 + elif U > v: + i += 1 + else: + intervals.append((max(u, U), min(v, V))) + if v < V: + i += 1 + else: + j += 1 + return IntervalSet(intervals) diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/reflection.py b/contrib/python/hypothesis/py3/hypothesis/internal/reflection.py index f64b527e45..31123b61ec 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/reflection.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/reflection.py @@ -27,7 +27,7 @@ from types import ModuleType from typing import TYPE_CHECKING, Any, Callable from unittest.mock import _patch as PatchType -from hypothesis.internal.compat import PYPY, is_typed_named_tuple, update_code_location +from hypothesis.internal.compat import PYPY, is_typed_named_tuple from hypothesis.utils.conventions import not_set from hypothesis.vendor.pretty import pretty @@ -61,7 +61,7 @@ def _clean_source(src: str) -> bytes: # lines - i.e. any decorators, so that adding `@example()`s keeps the same key. try: funcdef = ast.parse(src).body[0] - if sys.version_info[:2] == (3, 7) or (sys.version_info[:2] == (3, 8) and PYPY): + if sys.version_info[:2] == (3, 8) and PYPY: # We can't get a line number of the (async) def here, so as a best-effort # approximation we'll use str.split instead and hope for the best. tag = "async def " if isinstance(funcdef, ast.AsyncFunctionDef) else "def " @@ -88,13 +88,17 @@ def function_digest(function): multiple processes and is prone to changing significantly in response to minor changes to the function. - No guarantee of uniqueness though it usually will be. + No guarantee of uniqueness though it usually will be. Digest collisions + lead to unfortunate but not fatal problems during database replay. """ hasher = hashlib.sha384() try: src = inspect.getsource(function) except (OSError, TypeError): # If we can't actually get the source code, try for the name as a fallback. + # NOTE: We might want to change this to always adding function.__qualname__, + # to differentiate f.x. two classes having the same function implementation + # with class-dependent behaviour. try: hasher.update(function.__name__.encode()) except AttributeError: @@ -266,7 +270,7 @@ def is_first_param_referenced_in_function(f): tree = ast.parse(textwrap.dedent(inspect.getsource(f))) except Exception: return True # Assume it's OK unless we know otherwise - name = list(get_signature(f).parameters)[0] + name = next(iter(get_signature(f).parameters)) return any( isinstance(node, ast.Name) and node.id == name @@ -335,10 +339,10 @@ def extract_lambda_source(f): break except SyntaxError: continue - if tree is None and source.startswith("@"): - # This will always eventually find a valid expression because - # the decorator must be a valid Python function call, so will - # eventually be syntactically valid and break out of the loop. + if tree is None and source.startswith(("@", ".")): + # This will always eventually find a valid expression because the + # decorator or chained operator must be a valid Python function call, + # so will eventually be syntactically valid and break out of the loop. # Thus, this loop can never terminate normally. for i in range(len(source) + 1): p = source[1:i] @@ -420,8 +424,7 @@ def extract_lambda_source(f): source = WHITESPACE.sub(" ", source) source = SPACE_FOLLOWS_OPEN_BRACKET.sub("(", source) source = SPACE_PRECEDES_CLOSE_BRACKET.sub(")", source) - source = source.strip() - return source + return source.strip() def get_pretty_function_description(f): @@ -452,7 +455,7 @@ def nicerepr(v): return re.sub(r"(\[)~([A-Z][a-z]*\])", r"\g<1>\g<2>", pretty(v)) -def repr_call(f, args, kwargs, reorder=True): +def repr_call(f, args, kwargs, *, reorder=True): # Note: for multi-line pretty-printing, see RepresentationPrinter.repr_call() if reorder: args, kwargs = convert_positional_arguments(f, args, kwargs) @@ -520,7 +523,7 @@ def define_function_signature(name, docstring, signature): for a in signature.parameters: check_valid_identifier(a) - used_names = list(signature.parameters) + [name] + used_names = {*signature.parameters, name} newsig = signature.replace( parameters=[ @@ -608,8 +611,11 @@ def impersonate(target): """ def accept(f): - f.__code__ = update_code_location( - f.__code__, target.__code__.co_filename, target.__code__.co_firstlineno + # Lie shamelessly about where this code comes from, to hide the hypothesis + # internals from pytest, ipython, and other runtime introspection. + f.__code__ = f.__code__.replace( + co_filename=target.__code__.co_filename, + co_firstlineno=target.__code__.co_firstlineno, ) f.__name__ = target.__name__ f.__module__ = target.__module__ diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/scrutineer.py b/contrib/python/hypothesis/py3/hypothesis/internal/scrutineer.py index c3a53154c6..bf57728e45 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/scrutineer.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/scrutineer.py @@ -56,8 +56,10 @@ UNHELPFUL_LOCATIONS = ( f"{sep}re{sep}__init__.py", # refactored in Python 3.11 f"{sep}warnings.py", # Quite rarely, the first AFNP line is in Pytest's internals. + f"{sep}_pytest{sep}assertion{sep}__init__.py", f"{sep}_pytest{sep}assertion{sep}rewrite.py", f"{sep}_pytest{sep}_io{sep}saferepr.py", + f"{sep}pluggy{sep}_result.py", ) diff --git a/contrib/python/hypothesis/py3/hypothesis/provisional.py b/contrib/python/hypothesis/py3/hypothesis/provisional.py index 4502fea011..dec1abfc61 100644 --- a/contrib/python/hypothesis/py3/hypothesis/provisional.py +++ b/contrib/python/hypothesis/py3/hypothesis/provisional.py @@ -43,7 +43,7 @@ assert _comment.startswith("#") # Remove special-use domain names from the list. For more discussion # see https://github.com/HypothesisWorks/hypothesis/pull/3572 -TOP_LEVEL_DOMAINS = ["COM"] + sorted((d for d in _tlds if d != "ARPA"), key=len) +TOP_LEVEL_DOMAINS = ["COM", *sorted((d for d in _tlds if d != "ARPA"), key=len)] class DomainNameStrategy(st.SearchStrategy): diff --git a/contrib/python/hypothesis/py3/hypothesis/stateful.py b/contrib/python/hypothesis/py3/hypothesis/stateful.py index 69e1ac5b1b..a4017cc0d9 100644 --- a/contrib/python/hypothesis/py3/hypothesis/stateful.py +++ b/contrib/python/hypothesis/py3/hypothesis/stateful.py @@ -23,6 +23,7 @@ from io import StringIO from typing import ( Any, Callable, + ClassVar, Dict, Iterable, List, @@ -220,11 +221,9 @@ class StateMachineMeta(type): def __setattr__(cls, name, value): if name == "settings" and isinstance(value, Settings): raise AttributeError( - ( - "Assigning {cls}.settings = {value} does nothing. Assign " - "to {cls}.TestCase.settings, or use @{value} as a decorator " - "on the {cls} class." - ).format(cls=cls.__name__, value=value) + f"Assigning {cls.__name__}.settings = {value} does nothing. Assign " + f"to {cls.__name__}.TestCase.settings, or use @{value} as a decorator " + f"on the {cls.__name__} class." ) return super().__setattr__(name, value) @@ -239,9 +238,9 @@ class RuleBasedStateMachine(metaclass=StateMachineMeta): executed. """ - _rules_per_class: Dict[type, List[classmethod]] = {} - _invariants_per_class: Dict[type, List[classmethod]] = {} - _initializers_per_class: Dict[type, List[classmethod]] = {} + _rules_per_class: ClassVar[Dict[type, List[classmethod]]] = {} + _invariants_per_class: ClassVar[Dict[type, List[classmethod]]] = {} + _initializers_per_class: ClassVar[Dict[type, List[classmethod]]] = {} def __init__(self) -> None: if not self.rules(): @@ -387,7 +386,7 @@ class RuleBasedStateMachine(metaclass=StateMachineMeta): TestCase = TestCaseProperty() @classmethod - @lru_cache() + @lru_cache def _to_test_case(cls): class StateMachineTestCase(TestCase): settings = Settings(deadline=None, suppress_health_check=list(HealthCheck)) @@ -418,7 +417,7 @@ class Rule: if isinstance(v, Bundle): bundles.append(v) consume = isinstance(v, BundleConsumer) - arguments[k] = BundleReferenceStrategy(v.name, consume) + arguments[k] = BundleReferenceStrategy(v.name, consume=consume) else: arguments[k] = v self.bundles = tuple(bundles) @@ -429,7 +428,7 @@ self_strategy = st.runner() class BundleReferenceStrategy(SearchStrategy): - def __init__(self, name, consume=False): + def __init__(self, name: str, *, consume: bool = False): self.name = name self.consume = consume @@ -449,9 +448,9 @@ class BundleReferenceStrategy(SearchStrategy): class Bundle(SearchStrategy[Ex]): - def __init__(self, name: str, consume: bool = False) -> None: + def __init__(self, name: str, *, consume: bool = False) -> None: self.name = name - self.__reference_strategy = BundleReferenceStrategy(name, consume) + self.__reference_strategy = BundleReferenceStrategy(name, consume=consume) def do_draw(self, data): machine = data.draw(self_strategy) @@ -462,7 +461,7 @@ class Bundle(SearchStrategy[Ex]): consume = self.__reference_strategy.consume if consume is False: return f"Bundle(name={self.name!r})" - return f"Bundle(name={self.name!r}, consume={consume!r})" + return f"Bundle(name={self.name!r}, {consume=})" def calc_is_empty(self, recur): # We assume that a bundle will grow over time @@ -525,7 +524,7 @@ def _convert_targets(targets, target): if targets: raise InvalidArgument( "Passing both targets=%r and target=%r is redundant - pass " - "targets=%r instead." % (targets, target, tuple(targets) + (target,)) + "targets=%r instead." % (targets, target, (*targets, target)) ) targets = (target,) @@ -549,6 +548,7 @@ def _convert_targets(targets, target): "This will be an error in a future version of Hypothesis.", since="2021-09-08", has_codemod=False, + stacklevel=2, ) t = t.name converted_targets.append(t) @@ -801,19 +801,19 @@ def precondition(precond: Callable[[Any], bool]) -> Callable[[TestFunc], TestFun invariant = getattr(f, INVARIANT_MARKER, None) if rule is not None: assert invariant is None - new_rule = attr.evolve(rule, preconditions=rule.preconditions + (precond,)) + new_rule = attr.evolve(rule, preconditions=(*rule.preconditions, precond)) setattr(precondition_wrapper, RULE_MARKER, new_rule) elif invariant is not None: assert rule is None new_invariant = attr.evolve( - invariant, preconditions=invariant.preconditions + (precond,) + invariant, preconditions=(*invariant.preconditions, precond) ) setattr(precondition_wrapper, INVARIANT_MARKER, new_invariant) else: setattr( precondition_wrapper, PRECONDITIONS_MARKER, - getattr(f, PRECONDITIONS_MARKER, ()) + (precond,), + (*getattr(f, PRECONDITIONS_MARKER, ()), precond), ) return precondition_wrapper @@ -907,10 +907,7 @@ class RuleStrategy(SearchStrategy): ) def __repr__(self): - return "{}(machine={}({{...}}))".format( - self.__class__.__name__, - self.machine.__class__.__name__, - ) + return f"{self.__class__.__name__}(machine={self.machine.__class__.__name__}({{...}}))" def do_draw(self, data): if not any(self.is_valid(rule) for rule in self.rules): diff --git a/contrib/python/hypothesis/py3/hypothesis/statistics.py b/contrib/python/hypothesis/py3/hypothesis/statistics.py index c0b64434c6..33e4ea6706 100644 --- a/contrib/python/hypothesis/py3/hypothesis/statistics.py +++ b/contrib/python/hypothesis/py3/hypothesis/statistics.py @@ -11,6 +11,7 @@ import math from collections import Counter +from hypothesis._settings import Phase from hypothesis.utils.dynamicvariables import DynamicVariable collector = DynamicVariable(None) @@ -32,11 +33,11 @@ def describe_targets(best_targets): return [] elif len(best_targets) == 1: label, score = next(iter(best_targets.items())) - return [f"Highest target score: {score:g} (label={label!r})"] + return [f"Highest target score: {score:g} ({label=})"] else: lines = ["Highest target scores:"] for label, score in sorted(best_targets.items(), key=lambda x: x[::-1]): - lines.append(f"{score:>16g} (label={label!r})") + lines.append(f"{score:>16g} ({label=})") return lines @@ -73,7 +74,7 @@ def describe_statistics(stats_dict): """ lines = [stats_dict["nodeid"] + ":\n"] if "nodeid" in stats_dict else [] prev_failures = 0 - for phase in ["reuse", "generate", "shrink"]: + for phase in (p.name for p in list(Phase)[1:]): d = stats_dict.get(phase + "-phase", {}) # Basic information we report for every phase cases = d.get("test-cases", []) diff --git a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/collections.py b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/collections.py index 125054f4b0..5d9c9617aa 100644 --- a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/collections.py +++ b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/collections.py @@ -66,26 +66,26 @@ def tuples() -> SearchStrategy[Tuple[()]]: # pragma: no cover ... -@overload # noqa: F811 +@overload def tuples(__a1: SearchStrategy[Ex]) -> SearchStrategy[Tuple[Ex]]: # pragma: no cover ... -@overload # noqa: F811 +@overload def tuples( __a1: SearchStrategy[Ex], __a2: SearchStrategy[T] ) -> SearchStrategy[Tuple[Ex, T]]: # pragma: no cover ... -@overload # noqa: F811 +@overload def tuples( __a1: SearchStrategy[Ex], __a2: SearchStrategy[T], __a3: SearchStrategy[T3] ) -> SearchStrategy[Tuple[Ex, T, T3]]: # pragma: no cover ... -@overload # noqa: F811 +@overload def tuples( __a1: SearchStrategy[Ex], __a2: SearchStrategy[T], @@ -95,7 +95,7 @@ def tuples( ... -@overload # noqa: F811 +@overload def tuples( __a1: SearchStrategy[Ex], __a2: SearchStrategy[T], @@ -106,7 +106,7 @@ def tuples( ... -@overload # noqa: F811 +@overload def tuples( *args: SearchStrategy[Any], ) -> SearchStrategy[Tuple[Any, ...]]: # pragma: no cover @@ -115,7 +115,7 @@ def tuples( @cacheable @defines_strategy() -def tuples(*args: SearchStrategy[Any]) -> SearchStrategy[Tuple[Any, ...]]: # noqa: F811 +def tuples(*args: SearchStrategy[Any]) -> SearchStrategy[Tuple[Any, ...]]: """Return a strategy which generates a tuple of the same length as args by generating the value at index i from args[i]. @@ -234,12 +234,12 @@ class UniqueListStrategy(ListStrategy): while elements.more(): value = filtered.do_filtered_draw(data) if value is filter_not_satisfied: - elements.reject("Aborted test because unable to satisfy {filtered!r}") + elements.reject(f"Aborted test because unable to satisfy {filtered!r}") else: for key, seen in zip(self.keys, seen_sets): seen.add(key(value)) if self.tuple_suffixes is not None: - value = (value,) + data.draw(self.tuple_suffixes) + value = (value, *data.draw(self.tuple_suffixes)) result.append(value) assert self.max_size >= len(result) >= self.min_size return result @@ -271,7 +271,7 @@ class UniqueSampledListStrategy(UniqueListStrategy): for key, seen in zip(self.keys, seen_sets): seen.add(key(value)) if self.tuple_suffixes is not None: - value = (value,) + data.draw(self.tuple_suffixes) + value = (value, *data.draw(self.tuple_suffixes)) result.append(value) else: should_draw.reject( diff --git a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/core.py b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/core.py index 9764cde3c0..70a58866cb 100644 --- a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/core.py +++ b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/core.py @@ -8,6 +8,7 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. +import codecs import enum import math import operator @@ -17,29 +18,34 @@ import string import sys import typing import warnings +from contextvars import ContextVar from decimal import Context, Decimal, localcontext from fractions import Fraction from functools import lru_cache, reduce -from inspect import Parameter, Signature, isabstract, isclass, signature +from inspect import Parameter, Signature, isabstract, isclass from types import FunctionType from typing import ( - TYPE_CHECKING, Any, AnyStr, Callable, + Collection, Dict, FrozenSet, Hashable, Iterable, List, + Literal, Optional, Pattern, + Protocol, Sequence, Set, Tuple, Type, TypeVar, Union, + get_args, + get_origin, overload, ) from uuid import UUID @@ -55,7 +61,10 @@ from hypothesis.errors import ( SmallSearchSpaceWarning, ) from hypothesis.internal.cathetus import cathetus -from hypothesis.internal.charmap import as_general_categories +from hypothesis.internal.charmap import ( + as_general_categories, + categories as all_categories, +) from hypothesis.internal.compat import ( Concatenate, ParamSpec, @@ -113,6 +122,7 @@ from hypothesis.strategies._internal.recursive import RecursiveStrategy from hypothesis.strategies._internal.shared import SharedStrategy from hypothesis.strategies._internal.strategies import ( Ex, + Ex_Inv, SampledFromStrategy, T, one_of, @@ -128,20 +138,15 @@ from hypothesis.vendor.pretty import RepresentationPrinter if sys.version_info >= (3, 10): from types import EllipsisType as EllipsisType + from typing import TypeAlias as TypeAlias elif typing.TYPE_CHECKING: # pragma: no cover from builtins import ellipsis as EllipsisType + + from typing_extensions import TypeAlias else: EllipsisType = type(Ellipsis) # pragma: no cover -if sys.version_info >= (3, 8): - from typing import Protocol -elif TYPE_CHECKING: - from typing_extensions import Protocol -else: # pragma: no cover - Protocol = object - - @cacheable @defines_strategy() def booleans() -> SearchStrategy[bool]: @@ -158,20 +163,20 @@ def sampled_from(elements: Sequence[T]) -> SearchStrategy[T]: # pragma: no cove ... -@overload # noqa: F811 +@overload def sampled_from(elements: Type[enum.Enum]) -> SearchStrategy[Any]: # pragma: no cover # `SearchStrategy[Enum]` is unreliable due to metaclass issues. ... -@overload # noqa: F811 +@overload def sampled_from( elements: Union[Type[enum.Enum], Sequence[Any]] ) -> SearchStrategy[Any]: # pragma: no cover ... -@defines_strategy(try_non_lazy=True) # noqa: F811 +@defines_strategy(try_non_lazy=True) def sampled_from( elements: Union[Type[enum.Enum], Sequence[Any]] ) -> SearchStrategy[Any]: @@ -209,10 +214,13 @@ def sampled_from( raise InvalidArgument("Cannot sample from a length-zero sequence.") if len(values) == 1: return just(values[0]) - if isinstance(elements, type) and issubclass(elements, enum.Enum): - repr_ = f"sampled_from({elements.__module__}.{elements.__name__})" - else: - repr_ = f"sampled_from({elements!r})" + try: + if isinstance(elements, type) and issubclass(elements, enum.Enum): + repr_ = f"sampled_from({elements.__module__}.{elements.__name__})" + else: + repr_ = f"sampled_from({elements!r})" + except Exception: # pragma: no cover + repr_ = None if isclass(elements) and issubclass(elements, enum.Flag): # Combinations of enum.Flag members are also members. We generate # these dynamically, because static allocation takes O(2^n) memory. @@ -285,7 +293,7 @@ def lists( if unique_by is not None: if not (callable(unique_by) or isinstance(unique_by, tuple)): raise InvalidArgument( - f"unique_by={unique_by!r} is not a callable or tuple of callables" + f"{unique_by=} is not a callable or tuple of callables" ) if callable(unique_by): unique_by = (unique_by,) @@ -329,7 +337,7 @@ def lists( elements = SampledFromStrategy( sorted(range(elements.start, elements.end + 1), key=abs) if elements.end < 0 or elements.start > 0 - else list(range(0, elements.end + 1)) + else list(range(elements.end + 1)) + list(range(-1, elements.start - 1, -1)) ) @@ -337,7 +345,7 @@ def lists( element_count = len(elements.elements) if min_size > element_count: raise InvalidArgument( - f"Cannot create a collection of min_size={min_size!r} unique " + f"Cannot create a collection of {min_size=} unique " f"elements with values drawn from only {element_count} distinct " "elements" ) @@ -521,16 +529,64 @@ def dictionaries( ).map(dict_class) +# See https://en.wikipedia.org/wiki/Unicode_character_property#General_Category +CategoryName: "TypeAlias" = Literal[ + "L", # Letter + "Lu", # Letter, uppercase + "Ll", # Letter, lowercase + "Lt", # Letter, titlecase + "Lm", # Letter, modifier + "Lo", # Letter, other + "M", # Mark + "Mn", # Mark, nonspacing + "Mc", # Mark, spacing combining + "Me", # Mark, enclosing + "N", # Number + "Nd", # Number, decimal digit + "Nl", # Number, letter + "No", # Number, other + "P", # Punctuation + "Pc", # Punctuation, connector + "Pd", # Punctuation, dash + "Ps", # Punctuation, open + "Pe", # Punctuation, close + "Pi", # Punctuation, initial quote + "Pf", # Punctuation, final quote + "Po", # Punctuation, other + "S", # Symbol + "Sm", # Symbol, math + "Sc", # Symbol, currency + "Sk", # Symbol, modifier + "So", # Symbol, other + "Z", # Separator + "Zs", # Separator, space + "Zl", # Separator, line + "Zp", # Separator, paragraph + "C", # Other + "Cc", # Other, control + "Cf", # Other, format + "Cs", # Other, surrogate + "Co", # Other, private use + "Cn", # Other, not assigned +] + + @cacheable @defines_strategy(force_reusable_values=True) def characters( *, - whitelist_categories: Optional[Sequence[str]] = None, - blacklist_categories: Optional[Sequence[str]] = None, - blacklist_characters: Optional[Sequence[str]] = None, + codec: Optional[str] = None, min_codepoint: Optional[int] = None, max_codepoint: Optional[int] = None, - whitelist_characters: Optional[Sequence[str]] = None, + categories: Optional[Collection[CategoryName]] = None, + exclude_categories: Optional[Collection[CategoryName]] = None, + exclude_characters: Optional[Collection[str]] = None, + include_characters: Optional[Collection[str]] = None, + # Note: these arguments are deprecated aliases for backwards compatibility + blacklist_categories: Optional[Collection[CategoryName]] = None, + whitelist_categories: Optional[Collection[CategoryName]] = None, + blacklist_characters: Optional[Collection[str]] = None, + whitelist_characters: Optional[Collection[str]] = None, ) -> SearchStrategy[str]: r"""Generates characters, length-one :class:`python:str`\ ings, following specified filtering rules. @@ -538,19 +594,21 @@ def characters( - When no filtering rules are specified, any character can be produced. - If ``min_codepoint`` or ``max_codepoint`` is specified, then only characters having a codepoint in that range will be produced. - - If ``whitelist_categories`` is specified, then only characters from those + - If ``categories`` is specified, then only characters from those Unicode categories will be produced. This is a further restriction, characters must also satisfy ``min_codepoint`` and ``max_codepoint``. - - If ``blacklist_categories`` is specified, then any character from those - categories will not be produced. Any overlap between - ``whitelist_categories`` and ``blacklist_categories`` will raise an - exception, as each character can only belong to a single class. - - If ``whitelist_characters`` is specified, then any additional characters + - If ``exclude_categories`` is specified, then any character from those + categories will not be produced. You must not pass both ``categories`` + and ``exclude_categories``; these arguments are alternative ways to + specify exactly the same thing. + - If ``include_characters`` is specified, then any additional characters in that list will also be produced. - - If ``blacklist_characters`` is specified, then any characters in + - If ``exclude_characters`` is specified, then any characters in that list will be not be produced. Any overlap between - ``whitelist_characters`` and ``blacklist_characters`` will raise an + ``include_characters`` and ``exclude_characters`` will raise an exception. + - If ``codec`` is specified, only characters in the specified `codec encodings`_ + will be produced. The ``_codepoint`` arguments must be integers between zero and :obj:`python:sys.maxunicode`. The ``_characters`` arguments must be @@ -563,7 +621,17 @@ def characters( can be given to match all corresponding categories, for example ``'P'`` for characters in any punctuation category. + We allow codecs from the :mod:`codecs` module and their aliases, platform + specific and user-registered codecs if they are available, and + `python-specific text encodings`_ (but not text or binary transforms). + ``include_characters`` which cannot be encoded using this codec will + raise an exception. If non-encodable codepoints or categories are + explicitly allowed, the ``codec`` argument will exclude them without + raising an exception. + .. _general category: https://wikipedia.org/wiki/Unicode_character_property + .. _codec encodings: https://docs.python.org/3/library/codecs.html#encodings-and-unicode + .. _python-specific text encodings: https://docs.python.org/3/library/codecs.html#python-specific-encodings Examples from this strategy shrink towards the codepoint for ``'0'``, or the first allowable codepoint after it if ``'0'`` is excluded. @@ -571,62 +639,119 @@ def characters( check_valid_size(min_codepoint, "min_codepoint") check_valid_size(max_codepoint, "max_codepoint") check_valid_interval(min_codepoint, max_codepoint, "min_codepoint", "max_codepoint") + + if categories is not None and exclude_categories is not None: + raise InvalidArgument( + f"Pass at most one of {categories=} and {exclude_categories=} - " + "these arguments both specify which categories are allowed, so it " + "doesn't make sense to use both in a single call." + ) + + # Handle deprecation of whitelist/blacklist arguments + has_old_arg = any(v is not None for k, v in locals().items() if "list" in k) + has_new_arg = any(v is not None for k, v in locals().items() if "lude" in k) + if has_old_arg and has_new_arg: + raise InvalidArgument( + "The deprecated blacklist/whitelist arguments cannot be used in " + "the same call as their replacement include/exclude arguments." + ) + if blacklist_categories is not None: + exclude_categories = blacklist_categories + if whitelist_categories is not None: + categories = whitelist_categories + if blacklist_characters is not None: + exclude_characters = blacklist_characters + if whitelist_characters is not None: + include_characters = whitelist_characters + if ( min_codepoint is None and max_codepoint is None - and whitelist_categories is None - and blacklist_categories is None - and whitelist_characters is not None + and categories is None + and exclude_categories is None + and include_characters is not None + and codec is None ): raise InvalidArgument( "Nothing is excluded by other arguments, so passing only " - f"whitelist_characters={whitelist_characters!r} would have no effect. " - "Also pass whitelist_categories=(), or use " - f"sampled_from({whitelist_characters!r}) instead." + f"{include_characters=} would have no effect. " + "Also pass categories=(), or use " + f"sampled_from({include_characters!r}) instead." ) - blacklist_characters = blacklist_characters or "" - whitelist_characters = whitelist_characters or "" - overlap = set(blacklist_characters).intersection(whitelist_characters) + exclude_characters = exclude_characters or "" + include_characters = include_characters or "" + overlap = set(exclude_characters).intersection(include_characters) if overlap: raise InvalidArgument( f"Characters {sorted(overlap)!r} are present in both " - f"whitelist_characters={whitelist_characters!r}, and " - f"blacklist_characters={blacklist_characters!r}" + f"{include_characters=} and {exclude_characters=}" ) - blacklist_categories = as_general_categories( - blacklist_categories, "blacklist_categories" - ) - if ( - whitelist_categories is not None - and not whitelist_categories - and not whitelist_characters - ): + categories = as_general_categories(categories, "categories") + exclude_categories = as_general_categories(exclude_categories, "exclude_categories") + if categories is not None and not categories and not include_characters: raise InvalidArgument( - "When whitelist_categories is an empty collection and there are " - "no characters specified in whitelist_characters, nothing can " + "When `categories` is an empty collection and there are " + "no characters specified in include_characters, nothing can " "be generated by the characters() strategy." ) - whitelist_categories = as_general_categories( - whitelist_categories, "whitelist_categories" - ) - both_cats = set(blacklist_categories or ()).intersection(whitelist_categories or ()) + both_cats = set(exclude_categories or ()).intersection(categories or ()) if both_cats: + # Note: we check that exactly one of `categories` or `exclude_categories` is + # passed above, but retain this older check for the deprecated arguments. raise InvalidArgument( f"Categories {sorted(both_cats)!r} are present in both " - f"whitelist_categories={whitelist_categories!r}, and " - f"blacklist_categories={blacklist_categories!r}" + f"{categories=} and {exclude_categories=}" ) + elif exclude_categories is not None: + categories = set(all_categories()) - set(exclude_categories) + del exclude_categories - return OneCharStringStrategy( - whitelist_categories=whitelist_categories, - blacklist_categories=blacklist_categories, - blacklist_characters=blacklist_characters, + if codec is not None: + try: + codec = codecs.lookup(codec).name + # Check this is not a str-to-str or bytes-to-bytes codec; see + # https://docs.python.org/3/library/codecs.html#binary-transforms + "".encode(codec) + except LookupError: + raise InvalidArgument(f"{codec=} is not valid on this system") from None + except Exception: + raise InvalidArgument(f"{codec=} is not a valid codec") from None + + for char in include_characters: + try: + char.encode(encoding=codec, errors="strict") + except UnicodeEncodeError: + raise InvalidArgument( + f"Character {char!r} in {include_characters=} " + f"cannot be encoded with {codec=}" + ) from None + + # ascii and utf-8 are sufficient common that we have faster special handling + if codec == "ascii": + if (max_codepoint is None) or (max_codepoint > 127): + max_codepoint = 127 + codec = None + elif codec == "utf-8": + if categories is None: + categories = all_categories() + categories = tuple(c for c in categories if c != "Cs") + + return OneCharStringStrategy.from_characters_args( + categories=categories, + exclude_characters=exclude_characters, min_codepoint=min_codepoint, max_codepoint=max_codepoint, - whitelist_characters=whitelist_characters, + include_characters=include_characters, + codec=codec, ) +# Hide the deprecated aliases from documentation and casual inspection +characters.__signature__ = (__sig := get_signature(characters)).replace( # type: ignore + parameters=[p for p in __sig.parameters.values() if "list" not in p.name] +) + + # Cache size is limited by sys.maxunicode, but passing None makes it slightly faster. @lru_cache(maxsize=None) def _check_is_single_character(c): @@ -643,9 +768,7 @@ def _check_is_single_character(c): @cacheable @defines_strategy(force_reusable_values=True) def text( - alphabet: Union[Sequence[str], SearchStrategy[str]] = characters( - blacklist_categories=("Cs",) - ), + alphabet: Union[Collection[str], SearchStrategy[str]] = characters(codec="utf-8"), *, min_size: int = 0, max_size: Optional[int] = None, @@ -691,7 +814,7 @@ def text( f"which leads to violation of size constraints: {not_one_char!r}" ) char_strategy = ( - characters(whitelist_categories=(), whitelist_characters=alphabet) + characters(categories=(), include_characters=alphabet) if alphabet else nothing() ) @@ -700,10 +823,32 @@ def text( return TextStrategy(char_strategy, min_size=min_size, max_size=max_size) +@overload +def from_regex( + regex: Union[bytes, Pattern[bytes]], + *, + fullmatch: bool = False, +) -> SearchStrategy[bytes]: # pragma: no cover + ... + + +@overload +def from_regex( + regex: Union[str, Pattern[str]], + *, + fullmatch: bool = False, + alphabet: Union[str, SearchStrategy[str]] = characters(codec="utf-8"), +) -> SearchStrategy[str]: # pragma: no cover + ... + + @cacheable @defines_strategy() def from_regex( - regex: Union[AnyStr, Pattern[AnyStr]], *, fullmatch: bool = False + regex: Union[AnyStr, Pattern[AnyStr]], + *, + fullmatch: bool = False, + alphabet: Union[str, SearchStrategy[str], None] = None, ) -> SearchStrategy[AnyStr]: r"""Generates strings that contain a match for the given regex (i.e. ones for which :func:`python:re.search` will return a non-None result). @@ -729,15 +874,40 @@ def from_regex( Alternatively, passing ``fullmatch=True`` will ensure that the whole string is a match, as if you had used the ``\A`` and ``\Z`` markers. + The ``alphabet=`` argument constrains the characters in the generated + string, as for :func:`text`, and is only supported for unicode strings. + Examples from this strategy shrink towards shorter strings and lower character values, with exact behaviour that may depend on the pattern. """ + check_type((str, bytes, re.Pattern), regex, "regex") check_type(bool, fullmatch, "fullmatch") + pattern = regex.pattern if isinstance(regex, re.Pattern) else regex + if alphabet is not None: + check_type((str, SearchStrategy), alphabet, "alphabet") + if not isinstance(pattern, str): + raise InvalidArgument("alphabet= is not supported for bytestrings") + + if isinstance(alphabet, str): + alphabet = characters(categories=(), include_characters=alphabet) + char_strategy = unwrap_strategies(alphabet) + if isinstance(char_strategy, SampledFromStrategy): + alphabet = characters( + categories=(), + include_characters=alphabet.elements, # type: ignore + ) + elif not isinstance(char_strategy, OneCharStringStrategy): + raise InvalidArgument( + f"{alphabet=} must be a sampled_from() or characters() strategy" + ) + elif isinstance(pattern, str): + alphabet = characters(codec="utf-8") + # TODO: We would like to move this to the top level, but pending some major # refactoring it's hard to do without creating circular imports. from hypothesis.strategies._internal.regex import regex_strategy - return regex_strategy(regex, fullmatch) + return regex_strategy(regex, fullmatch, alphabet=alphabet) @cacheable @@ -783,6 +953,9 @@ def randoms( would occur with very low probability when it is set to True, and this flag should only be set to True when your code relies on the distribution of values for correctness. + + For managing global state, see the :func:`~hypothesis.strategies.random_module` + strategy and :func:`~hypothesis.register_random` function. """ check_type(bool, note_method_calls, "note_method_calls") check_type(bool, use_true_random, "use_true_random") @@ -818,16 +991,16 @@ class RandomModule(SearchStrategy): @cacheable @defines_strategy() def random_module() -> SearchStrategy[RandomSeeder]: - """The Hypothesis engine handles PRNG state for the stdlib and Numpy random - modules internally, always seeding them to zero and restoring the previous - state after the test. + """Hypothesis always seeds global PRNGs before running a test, and restores the + previous state afterwards. If having a fixed seed would unacceptably weaken your tests, and you cannot use a ``random.Random`` instance provided by :func:`~hypothesis.strategies.randoms`, this strategy calls :func:`python:random.seed` with an arbitrary integer and passes you an opaque object whose repr displays the seed value for debugging. - If ``numpy.random`` is available, that state is also managed. + If ``numpy.random`` is available, that state is also managed, as is anything + managed by :func:`hypothesis.register_random`. Examples from these strategy shrink to seeds closer to zero. """ @@ -888,14 +1061,12 @@ class BuildsStrategy(SearchStrategy): return f"builds({', '.join(bits)})" -# The ideal signature builds(target, /, *args, **kwargs) is unfortunately a -# SyntaxError before Python 3.8 so we emulate it with manual argument unpacking. -# Note that for the benefit of documentation and introspection tools, we set the -# __signature__ attribute to show the semantic rather than actual signature. @cacheable @defines_strategy() def builds( - *callable_and_args: Union[Callable[..., Ex], SearchStrategy[Any]], + target: Callable[..., Ex], + /, + *args: SearchStrategy[Any], **kwargs: Union[SearchStrategy[Any], EllipsisType], ) -> SearchStrategy[Ex]: """Generates values by drawing from ``args`` and ``kwargs`` and passing @@ -919,12 +1090,6 @@ def builds( Examples from this strategy shrink by shrinking the argument values to the callable. """ - if not callable_and_args: - raise InvalidArgument( # pragma: no cover - "builds() must be passed a callable as the first positional " - "argument, but no positional arguments were given." - ) - target, args = callable_and_args[0], callable_and_args[1:] if not callable(target): raise InvalidArgument( "The first positional argument to builds() must be a callable " @@ -958,44 +1123,21 @@ def builds( from hypothesis.strategies._internal.types import _global_type_lookup for kw, t in infer_for.items(): - if ( - getattr(t, "__module__", None) in ("builtins", "typing") - or t in _global_type_lookup - ): + if t in _global_type_lookup: kwargs[kw] = from_type(t) else: # We defer resolution of these type annotations so that the obvious - # approach to registering recursive types just works. See + # approach to registering recursive types just works. I.e., + # if we're inside `register_type_strategy(cls, builds(cls, ...))` + # and `...` contains recursion on `cls`. See # https://github.com/HypothesisWorks/hypothesis/issues/3026 kwargs[kw] = deferred(lambda t=t: from_type(t)) # type: ignore return BuildsStrategy(target, args, kwargs) -if sys.version_info[:2] >= (3, 8): - # See notes above definition - this signature is compatible and better - # matches the semantics of the function. Great for documentation! - sig = signature(builds) - args, kwargs = sig.parameters.values() - builds = define_function_signature( - name=builds.__name__, - docstring=builds.__doc__, - signature=sig.replace( - parameters=[ - Parameter( - name="target", - kind=Parameter.POSITIONAL_ONLY, - annotation=Callable[..., Ex], - ), - args.replace(name="args", annotation=SearchStrategy[Any]), - kwargs, - ] - ), - )(builds) - - @cacheable @defines_strategy(never_lazy=True) -def from_type(thing: Type[Ex]) -> SearchStrategy[Ex]: +def from_type(thing: Type[Ex_Inv]) -> SearchStrategy[Ex_Inv]: """Looks up the appropriate search strategy for the given type. ``from_type`` is used internally to fill in missing arguments to @@ -1050,7 +1192,7 @@ def from_type(thing: Type[Ex]) -> SearchStrategy[Ex]: try: with warnings.catch_warnings(): warnings.simplefilter("error") - return _from_type(thing, []) + return _from_type(thing) except Exception: return _from_type_deferred(thing) @@ -1060,20 +1202,32 @@ def _from_type_deferred(thing: Type[Ex]) -> SearchStrategy[Ex]: # underlying strategy wherever possible, as a form of user education, but # would prefer to fall back to the default "from_type(...)" repr instead of # "deferred(...)" for recursive types or invalid arguments. + try: + thing_repr = nicerepr(thing) + if hasattr(thing, "__module__"): + module_prefix = f"{thing.__module__}." + if not thing_repr.startswith(module_prefix): + thing_repr = module_prefix + thing_repr + repr_ = f"from_type({thing_repr})" + except Exception: # pragma: no cover + repr_ = None return LazyStrategy( - lambda thing: deferred(lambda: _from_type(thing, [])), + lambda thing: deferred(lambda: _from_type(thing)), (thing,), {}, - force_repr=f"from_type({thing!r})", + force_repr=repr_, ) -def _from_type(thing: Type[Ex], recurse_guard: List[Type[Ex]]) -> SearchStrategy[Ex]: +_recurse_guard: ContextVar = ContextVar("recurse_guard") + + +def _from_type(thing: Type[Ex]) -> SearchStrategy[Ex]: # TODO: We would like to move this to the top level, but pending some major # refactoring it's hard to do without creating circular imports. from hypothesis.strategies._internal import types - def as_strategy(strat_or_callable, thing, final=True): + def as_strategy(strat_or_callable, thing): # User-provided strategies need some validation, and callables even more # of it. We do this in three places, hence the helper function if not isinstance(strat_or_callable, SearchStrategy): @@ -1081,6 +1235,8 @@ def _from_type(thing: Type[Ex], recurse_guard: List[Type[Ex]]) -> SearchStrategy strategy = strat_or_callable(thing) else: strategy = strat_or_callable + if strategy is NotImplemented: + return NotImplemented if not isinstance(strategy, SearchStrategy): raise ResolutionFailed( f"Error: {thing} was registered for {nicerepr(strat_or_callable)}, " @@ -1092,11 +1248,17 @@ def _from_type(thing: Type[Ex], recurse_guard: List[Type[Ex]]) -> SearchStrategy def from_type_guarded(thing): """Returns the result of producer, or ... if recursion on thing is encountered""" + try: + recurse_guard = _recurse_guard.get() + except LookupError: + # We can't simply define the contextvar with default=[], as the + # default object would be shared across contexts + _recurse_guard.set(recurse_guard := []) if thing in recurse_guard: raise RewindRecursive(thing) recurse_guard.append(thing) try: - return _from_type(thing, recurse_guard) + return _from_type(thing) except RewindRecursive as rr: if rr.target != thing: raise @@ -1117,12 +1279,21 @@ def _from_type(thing: Type[Ex], recurse_guard: List[Type[Ex]]) -> SearchStrategy # Check if we have an explicitly registered strategy for this thing, # resolve it so, and otherwise resolve as for the base type. if thing in types._global_type_lookup: - return as_strategy(types._global_type_lookup[thing], thing) - return _from_type(thing.__supertype__, recurse_guard) + strategy = as_strategy(types._global_type_lookup[thing], thing) + if strategy is not NotImplemented: + return strategy + return _from_type(thing.__supertype__) # Unions are not instances of `type` - but we still want to resolve them! if types.is_a_union(thing): args = sorted(thing.__args__, key=types.type_sorting_key) - return one_of([_from_type(t, recurse_guard) for t in args]) + return one_of([_from_type(t) for t in args]) + # We also have a special case for TypeVars. + # They are represented as instances like `~T` when they come here. + # We need to work with their type instead. + if isinstance(thing, TypeVar) and type(thing) in types._global_type_lookup: + strategy = as_strategy(types._global_type_lookup[type(thing)], thing) + if strategy is not NotImplemented: + return strategy if not types.is_a_type(thing): if isinstance(thing, str): # See https://github.com/HypothesisWorks/hypothesis/issues/3016 @@ -1132,7 +1303,7 @@ def _from_type(thing: Type[Ex], recurse_guard: List[Type[Ex]]) -> SearchStrategy "`from __future__ import annotations` instead of forward-reference " "strings." ) - raise InvalidArgument(f"thing={thing!r} must be a type") # pragma: no cover + raise InvalidArgument(f"{thing=} must be a type") # pragma: no cover if thing in types.NON_RUNTIME_TYPES: # Some code like `st.from_type(TypeAlias)` does not make sense. # Because there are types in python that do not exist in runtime. @@ -1147,7 +1318,9 @@ def _from_type(thing: Type[Ex], recurse_guard: List[Type[Ex]]) -> SearchStrategy # convert empty results into an explicit error. try: if thing in types._global_type_lookup: - return as_strategy(types._global_type_lookup[thing], thing) + strategy = as_strategy(types._global_type_lookup[thing], thing) + if strategy is not NotImplemented: + return strategy except TypeError: # pragma: no cover # This is due to a bizarre divergence in behaviour under Python 3.9.0: # typing.Callable[[], foo] has __args__ = (foo,) but collections.abc.Callable @@ -1165,7 +1338,7 @@ def _from_type(thing: Type[Ex], recurse_guard: List[Type[Ex]]) -> SearchStrategy optional = set(getattr(thing, "__optional_keys__", ())) anns = {} for k, v in get_type_hints(thing).items(): - origin = getattr(v, "__origin__", None) + origin = get_origin(v) if origin in types.RequiredTypes + types.NotRequiredTypes: if origin in types.NotRequiredTypes: optional.add(k) @@ -1190,11 +1363,6 @@ def _from_type(thing: Type[Ex], recurse_guard: List[Type[Ex]]) -> SearchStrategy mapping={k: v for k, v in anns.items() if k not in optional}, optional={k: v for k, v in anns.items() if k in optional}, ) - # We also have a special case for TypeVars. - # They are represented as instances like `~T` when they come here. - # We need to work with their type instead. - if isinstance(thing, TypeVar) and type(thing) in types._global_type_lookup: - return as_strategy(types._global_type_lookup[type(thing)], thing) # If there's no explicitly registered strategy, maybe a subtype of thing # is registered - if so, we can resolve it to the subclass strategy. @@ -1203,8 +1371,8 @@ def _from_type(thing: Type[Ex], recurse_guard: List[Type[Ex]]) -> SearchStrategy # subclass and instance checks. if isinstance(thing, types.typing_root_type) or ( sys.version_info[:2] >= (3, 9) - and isinstance(getattr(thing, "__origin__", None), type) - and getattr(thing, "__args__", None) + and isinstance(get_origin(thing), type) + and get_args(thing) ): return types.from_typing_type(thing) # If it's not from the typing module, we get all registered types that are @@ -1212,11 +1380,16 @@ def _from_type(thing: Type[Ex], recurse_guard: List[Type[Ex]]) -> SearchStrategy # type. For example, `Number -> integers() | floats()`, but bools() is # not included because bool is a subclass of int as well as Number. strategies = [ - as_strategy(v, thing, final=False) - for k, v in sorted(types._global_type_lookup.items(), key=repr) - if isinstance(k, type) - and issubclass(k, thing) - and sum(types.try_issubclass(k, typ) for typ in types._global_type_lookup) == 1 + s + for s in ( + as_strategy(v, thing) + for k, v in sorted(types._global_type_lookup.items(), key=repr) + if isinstance(k, type) + and issubclass(k, thing) + and sum(types.try_issubclass(k, typ) for typ in types._global_type_lookup) + == 1 + ) + if s is not NotImplemented ] if any(not s.is_empty for s in strategies): return one_of(strategies) @@ -1266,6 +1439,7 @@ def _from_type(thing: Type[Ex], recurse_guard: List[Type[Ex]]) -> SearchStrategy "to resolve to a strategy which can generate more than one value, " "or silence this warning.", SmallSearchSpaceWarning, + stacklevel=2, ) return builds(thing, **kwargs) # And if it's an abstract type, we'll resolve to a union of subclasses instead. @@ -1278,13 +1452,13 @@ def _from_type(thing: Type[Ex], recurse_guard: List[Type[Ex]]) -> SearchStrategy subclass_strategies = nothing() for sc in subclasses: try: - subclass_strategies |= _from_type(sc, recurse_guard) + subclass_strategies |= _from_type(sc) except Exception: pass if subclass_strategies.is_empty: # We're unable to resolve subclasses now, but we might be able to later - # so we'll just go back to the mixed distribution. - return sampled_from(subclasses).flatmap(lambda t: _from_type(t, recurse_guard)) + return sampled_from(subclasses).flatmap(_from_type) return subclass_strategies @@ -1321,16 +1495,16 @@ def fractions( if max_denominator is not None: if max_denominator < 1: - raise InvalidArgument(f"max_denominator={max_denominator!r} must be >= 1") + raise InvalidArgument(f"{max_denominator=} must be >= 1") if min_value is not None and min_value.denominator > max_denominator: raise InvalidArgument( - f"The min_value={min_value!r} has a denominator greater than the " - f"max_denominator={max_denominator!r}" + f"The {min_value=} has a denominator greater than the " + f"{max_denominator=}" ) if max_value is not None and max_value.denominator > max_denominator: raise InvalidArgument( - f"The max_value={max_value!r} has a denominator greater than the " - f"max_denominator={max_denominator!r}" + f"The {max_value=} has a denominator greater than the " + f"{max_denominator=}" ) if min_value is not None and min_value == max_value: @@ -1390,9 +1564,7 @@ def _as_finite_decimal( if value.is_infinite() and (value < 0 if "min" in name else value > 0): if allow_infinity or allow_infinity is None: return None - raise InvalidArgument( - f"allow_infinity={allow_infinity!r}, but {name}={value!r}" - ) + raise InvalidArgument(f"{allow_infinity=}, but {name}={value!r}") # This could be infinity, quiet NaN, or signalling NaN raise InvalidArgument(f"Invalid {name}={value!r}") @@ -1430,7 +1602,7 @@ def decimals( # Convert min_value and max_value to Decimal values, and validate args check_valid_integer(places, "places") if places is not None and places < 0: - raise InvalidArgument(f"places={places!r} may not be negative") + raise InvalidArgument(f"{places=} may not be negative") min_value = _as_finite_decimal(min_value, "min_value", allow_infinity) max_value = _as_finite_decimal(max_value, "max_value", allow_infinity) check_valid_interval(min_value, max_value, "min_value", "max_value") @@ -1460,7 +1632,7 @@ def decimals( if min_num is not None and max_num is not None and min_num > max_num: raise InvalidArgument( f"There are no decimals with {places} places between " - f"min_value={min_value!r} and max_value={max_value!r}" + f"{min_value=} and {max_value=}" ) strat = integers(min_num, max_num).map(int_to_decimal) else: @@ -1578,7 +1750,7 @@ class DrawFn(Protocol): def __init__(self): raise TypeError("Protocols cannot be instantiated") # pragma: no cover - # On Python 3.8+, Protocol overrides our signature for __init__, + # Protocol overrides our signature for __init__, # so we override it right back to make the docs look nice. __signature__: Signature = Signature(parameters=[]) @@ -1679,7 +1851,7 @@ def complex_numbers( allow_infinity: Optional[bool] = None, allow_nan: Optional[bool] = None, allow_subnormal: bool = True, - width: int = 128, + width: Literal[32, 64, 128] = 128, ) -> SearchStrategy[complex]: """Returns a strategy that generates :class:`~python:complex` numbers. @@ -1726,21 +1898,17 @@ def complex_numbers( if allow_infinity is None: allow_infinity = bool(max_magnitude is None) elif allow_infinity and max_magnitude is not None: - raise InvalidArgument( - f"Cannot have allow_infinity={allow_infinity!r} with " - f"max_magnitude={max_magnitude!r}" - ) + raise InvalidArgument(f"Cannot have {allow_infinity=} with {max_magnitude=}") if allow_nan is None: allow_nan = bool(min_magnitude == 0 and max_magnitude is None) elif allow_nan and not (min_magnitude == 0 and max_magnitude is None): raise InvalidArgument( - f"Cannot have allow_nan={allow_nan!r}, min_magnitude={min_magnitude!r} " - f"max_magnitude={max_magnitude!r}" + f"Cannot have {allow_nan=}, {min_magnitude=}, {max_magnitude=}" ) check_type(bool, allow_subnormal, "allow_subnormal") if width not in (32, 64, 128): raise InvalidArgument( - f"width={width!r}, but must be 32, 64 or 128 (other complex dtypes " + f"{width=}, but must be 32, 64 or 128 (other complex dtypes " "such as complex192 or complex256 are not supported)" # For numpy, these types would be supported (but not by CPython): # https://numpy.org/doc/stable/reference/arrays.scalars.html#complex-floating-point-types @@ -1836,7 +2004,7 @@ def _maybe_nil_uuids(draw, uuid): @cacheable @defines_strategy(force_reusable_values=True) def uuids( - *, version: Optional[int] = None, allow_nil: bool = False + *, version: Optional[Literal[1, 2, 3, 4, 5]] = None, allow_nil: bool = False ) -> SearchStrategy[UUID]: """Returns a strategy that generates :class:`UUIDs <uuid.UUID>`. @@ -1853,7 +2021,7 @@ def uuids( check_type(bool, allow_nil, "allow_nil") if version not in (None, 1, 2, 3, 4, 5): raise InvalidArgument( - f"version={version!r}, but version must be in " + f"{version=}, but version must be in " "(None, 1, 2, 3, 4, 5) to pass to the uuid.UUID constructor." ) random_uuids = shared( @@ -1987,7 +2155,10 @@ def register_type_strategy( for an argument with a default value. ``strategy`` may be a search strategy, or a function that takes a type and - returns a strategy (useful for generic types). + returns a strategy (useful for generic types). The function may return + :data:`NotImplemented` to conditionally not provide a strategy for the type + (the type will still be resolved by other methods, if possible, as if the + function was not registered). Note that you may not register a parametrised generic type (such as ``MyCollection[int]``) directly, because the resolution logic does not @@ -2000,26 +2171,25 @@ def register_type_strategy( from hypothesis.strategies._internal import types if not types.is_a_type(custom_type): - raise InvalidArgument(f"custom_type={custom_type!r} must be a type") + raise InvalidArgument(f"{custom_type=} must be a type") if custom_type in types.NON_RUNTIME_TYPES: raise InvalidArgument( - f"custom_type={custom_type!r} is not allowed to be registered, " + f"{custom_type=} is not allowed to be registered, " f"because there is no such thing as a runtime instance of {custom_type!r}" ) elif not (isinstance(strategy, SearchStrategy) or callable(strategy)): raise InvalidArgument( - "strategy=%r must be a SearchStrategy, or a function that takes " + f"{strategy=} must be a SearchStrategy, or a function that takes " "a generic type and returns a specific SearchStrategy" ) elif isinstance(strategy, SearchStrategy) and strategy.is_empty: - raise InvalidArgument("strategy=%r must not be empty") + raise InvalidArgument(f"{strategy=} must not be empty") elif types.has_type_arguments(custom_type): - origin = getattr(custom_type, "__origin__", None) raise InvalidArgument( f"Cannot register generic type {custom_type!r}, because it has type " "arguments which would not be handled. Instead, register a function " - f"for {origin!r} which can inspect specific type objects and return a " - "strategy." + f"for {get_origin(custom_type)!r} which can inspect specific type " + "objects and return a strategy." ) if ( "pydantic.generics" in sys.modules diff --git a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/datetime.py b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/datetime.py index c3501d1506..a23779711f 100644 --- a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/datetime.py +++ b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/datetime.py @@ -9,10 +9,10 @@ # obtain one at https://mozilla.org/MPL/2.0/. import datetime as dt -import os.path from calendar import monthrange from functools import lru_cache from importlib import resources +from pathlib import Path from typing import Optional from hypothesis.errors import InvalidArgument @@ -215,13 +215,13 @@ def datetimes( check_type(dt.datetime, min_value, "min_value") check_type(dt.datetime, max_value, "max_value") if min_value.tzinfo is not None: - raise InvalidArgument(f"min_value={min_value!r} must not have tzinfo") + raise InvalidArgument(f"{min_value=} must not have tzinfo") if max_value.tzinfo is not None: - raise InvalidArgument(f"max_value={max_value!r} must not have tzinfo") + raise InvalidArgument(f"{max_value=} must not have tzinfo") check_valid_interval(min_value, max_value, "min_value", "max_value") if not isinstance(timezones, SearchStrategy): raise InvalidArgument( - f"timezones={timezones!r} must be a SearchStrategy that can " + f"{timezones=} must be a SearchStrategy that can " "provide tzinfo for datetimes (either None or dt.tzinfo objects)" ) return DatetimeStrategy(min_value, max_value, timezones, allow_imaginary) @@ -258,9 +258,9 @@ def times( check_type(dt.time, min_value, "min_value") check_type(dt.time, max_value, "max_value") if min_value.tzinfo is not None: - raise InvalidArgument(f"min_value={min_value!r} must not have tzinfo") + raise InvalidArgument(f"{min_value=} must not have tzinfo") if max_value.tzinfo is not None: - raise InvalidArgument(f"max_value={max_value!r} must not have tzinfo") + raise InvalidArgument(f"{max_value=} must not have tzinfo") check_valid_interval(min_value, max_value, "min_value", "max_value") return TimeStrategy(min_value, max_value, timezones) @@ -342,7 +342,7 @@ def timedeltas( def _valid_key_cacheable(tzpath, key): assert isinstance(tzpath, tuple) # zoneinfo changed, better update this function! for root in tzpath: - if os.path.exists(os.path.join(root, key)): # pragma: no branch + if Path(root).joinpath(key).exists(): # pragma: no branch # No branch because most systems only have one TZPATH component. return True else: # pragma: no cover @@ -411,7 +411,7 @@ def timezone_keys( "Run `pip install hypothesis[zoneinfo]` and try again." ) - available_timezones = ("UTC",) + tuple(sorted(zoneinfo.available_timezones())) + available_timezones = ("UTC", *sorted(zoneinfo.available_timezones())) # TODO: filter out alias and deprecated names if disallowed diff --git a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/featureflags.py b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/featureflags.py index 5976168212..66d648cce1 100644 --- a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/featureflags.py +++ b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/featureflags.py @@ -96,7 +96,7 @@ class FeatureFlags: disabled.append(name) else: enabled.append(name) - return f"FeatureFlags(enabled={enabled!r}, disabled={disabled!r})" + return f"FeatureFlags({enabled=}, {disabled=})" class FeatureStrategy(SearchStrategy): diff --git a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/flatmapped.py b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/flatmapped.py index b1fca8f883..49cb4e0cdd 100644 --- a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/flatmapped.py +++ b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/flatmapped.py @@ -23,10 +23,7 @@ class FlatMapStrategy(SearchStrategy): def __repr__(self): if not hasattr(self, "_cached_repr"): - self._cached_repr = "{!r}.flatmap({})".format( - self.flatmapped_strategy, - get_pretty_function_description(self.expand), - ) + self._cached_repr = f"{self.flatmapped_strategy!r}.flatmap({get_pretty_function_description(self.expand)})" return self._cached_repr def do_draw(self, data): diff --git a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/ipaddress.py b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/ipaddress.py index 717943f29e..75aaaba8d7 100644 --- a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/ipaddress.py +++ b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/ipaddress.py @@ -9,7 +9,7 @@ # obtain one at https://mozilla.org/MPL/2.0/. from ipaddress import IPv4Address, IPv4Network, IPv6Address, IPv6Network, ip_network -from typing import Optional, Union +from typing import Literal, Optional, Union from hypothesis.errors import InvalidArgument from hypothesis.internal.validation import check_type @@ -73,7 +73,7 @@ SPECIAL_IPv6_RANGES = ( @defines_strategy(force_reusable_values=True) def ip_addresses( *, - v: Optional[int] = None, + v: Optional[Literal[4, 6]] = None, network: Optional[Union[str, IPv4Network, IPv6Network]] = None, ) -> SearchStrategy[Union[IPv4Address, IPv6Address]]: r"""Generate IP addresses - ``v=4`` for :class:`~python:ipaddress.IPv4Address`\ es, @@ -93,7 +93,7 @@ def ip_addresses( if v is not None: check_type(int, v, "v") if v not in (4, 6): - raise InvalidArgument(f"v={v!r}, but only v=4 or v=6 are valid") + raise InvalidArgument(f"{v=}, but only v=4 or v=6 are valid") if network is None: # We use the reserved-address registries to boost the chance # of generating one of the various special types of address. @@ -113,6 +113,6 @@ def ip_addresses( check_type((IPv4Network, IPv6Network), network, "network") assert isinstance(network, (IPv4Network, IPv6Network)) # for Mypy if v not in (None, network.version): - raise InvalidArgument(f"v={v!r} is incompatible with network={network!r}") + raise InvalidArgument(f"{v=} is incompatible with {network=}") addr_type = IPv4Address if network.version == 4 else IPv6Address return integers(int(network[0]), int(network[-1])).map(addr_type) # type: ignore diff --git a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/lazy.py b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/lazy.py index 18292c044c..5e493a9099 100644 --- a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/lazy.py +++ b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/lazy.py @@ -117,12 +117,16 @@ class LazyStrategy(SearchStrategy): return self.__wrapped_strategy def filter(self, condition): + try: + repr_ = f"{self!r}{_repr_filter(condition)}" + except Exception: + repr_ = None return LazyStrategy( self.function, self.__args, self.__kwargs, - self.__filters + (condition,), - force_repr=f"{self!r}{_repr_filter(condition)}", + (*self.__filters, condition), + force_repr=repr_, ) def do_validate(self): diff --git a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/numbers.py b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/numbers.py index 825e91de2e..ae66a2c1b2 100644 --- a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/numbers.py +++ b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/numbers.py @@ -12,7 +12,7 @@ import math from decimal import Decimal from fractions import Fraction from sys import float_info -from typing import Optional, Union +from typing import Literal, Optional, Union from hypothesis.control import reject from hypothesis.errors import InvalidArgument @@ -168,7 +168,8 @@ def integers( SMALLEST_SUBNORMAL = next_up(0.0) SIGNALING_NAN = int_to_float(0x7FF8_0000_0000_0001) # nonzero mantissa -assert math.isnan(SIGNALING_NAN) and math.copysign(1, SIGNALING_NAN) == 1 +assert math.isnan(SIGNALING_NAN) +assert math.copysign(1, SIGNALING_NAN) == 1 NASTY_FLOATS = sorted( [ @@ -263,11 +264,15 @@ class FloatStrategy(SearchStrategy): if _sign_aware_lte(0.0, max_value): pos_min = max(min_value, smallest_nonzero_magnitude) allow_zero = _sign_aware_lte(min_value, 0.0) - self.pos_clamper = make_float_clamper(pos_min, max_value, allow_zero) + self.pos_clamper = make_float_clamper( + pos_min, max_value, allow_zero=allow_zero + ) if _sign_aware_lte(min_value, -0.0): neg_max = min(max_value, -smallest_nonzero_magnitude) allow_zero = _sign_aware_lte(-0.0, max_value) - self.neg_clamper = make_float_clamper(-neg_max, -min_value, allow_zero) + self.neg_clamper = make_float_clamper( + -neg_max, -min_value, allow_zero=allow_zero + ) self.forced_sign_bit: Optional[int] = None if (self.pos_clamper is None) != (self.neg_clamper is None): @@ -378,7 +383,7 @@ def floats( allow_nan: Optional[bool] = None, allow_infinity: Optional[bool] = None, allow_subnormal: Optional[bool] = None, - width: int = 64, + width: Literal[16, 32, 64] = 64, exclude_min: bool = False, exclude_max: bool = False, ) -> SearchStrategy[float]: @@ -423,13 +428,11 @@ def floats( if allow_nan is None: allow_nan = bool(min_value is None and max_value is None) elif allow_nan and (min_value is not None or max_value is not None): - raise InvalidArgument( - f"Cannot have allow_nan={allow_nan!r}, with min_value or max_value" - ) + raise InvalidArgument(f"Cannot have {allow_nan=}, with min_value or max_value") if width not in (16, 32, 64): raise InvalidArgument( - f"Got width={width!r}, but the only valid values " + f"Got {width=}, but the only valid values " "are the integers 16, 32, and 64." ) @@ -472,7 +475,7 @@ def floats( "writeup - and good luck!" ) raise FloatingPointError( - f"Got allow_subnormal={allow_subnormal!r}, but we can't represent " + f"Got {allow_subnormal=}, but we can't represent " f"subnormal floats right now, in violation of the IEEE-754 floating-point " f"specification. {ftz_msg}" ) @@ -488,18 +491,18 @@ def floats( if min_value != min_arg: raise InvalidArgument( f"min_value={min_arg!r} cannot be exactly represented as a float " - f"of width {width} - use min_value={min_value!r} instead." + f"of width {width} - use {min_value=} instead." ) if max_value != max_arg: raise InvalidArgument( f"max_value={max_arg!r} cannot be exactly represented as a float " - f"of width {width} - use max_value={max_value!r} instead." + f"of width {width} - use {max_value=} instead." ) if exclude_min and (min_value is None or min_value == math.inf): - raise InvalidArgument(f"Cannot exclude min_value={min_value!r}") + raise InvalidArgument(f"Cannot exclude {min_value=}") if exclude_max and (max_value is None or max_value == -math.inf): - raise InvalidArgument(f"Cannot exclude max_value={max_value!r}") + raise InvalidArgument(f"Cannot exclude {max_value=}") assumed_allow_subnormal = allow_subnormal is None or allow_subnormal if min_value is not None and ( @@ -508,7 +511,8 @@ def floats( min_value = next_up_normal(min_value, width, assumed_allow_subnormal) if min_value == min_arg: assert min_value == min_arg == 0 - assert is_negative(min_arg) and not is_negative(min_value) + assert is_negative(min_arg) + assert not is_negative(min_value) min_value = next_up_normal(min_value, width, assumed_allow_subnormal) assert min_value > min_arg # type: ignore if max_value is not None and ( @@ -517,7 +521,8 @@ def floats( max_value = next_down_normal(max_value, width, assumed_allow_subnormal) if max_value == max_arg: assert max_value == max_arg == 0 - assert is_negative(max_value) and not is_negative(max_arg) + assert is_negative(max_value) + assert not is_negative(max_arg) max_value = next_down_normal(max_value, width, assumed_allow_subnormal) assert max_value < max_arg # type: ignore @@ -543,7 +548,7 @@ def floats( "and max_value=%r" % (width, min_arg, max_arg) ) if exclude_min or exclude_max: - msg += f", exclude_min={exclude_min!r} and exclude_max={exclude_max!r}" + msg += f", {exclude_min=} and {exclude_max=}" raise InvalidArgument(msg) if allow_infinity is None: @@ -551,8 +556,7 @@ def floats( elif allow_infinity: if min_value is not None and max_value is not None: raise InvalidArgument( - f"Cannot have allow_infinity={allow_infinity!r}, " - "with both min_value and max_value" + f"Cannot have {allow_infinity=}, with both min_value and max_value" ) elif min_value == math.inf: if min_arg == math.inf: diff --git a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/random.py b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/random.py index c9bf4550a2..882ed106e5 100644 --- a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/random.py +++ b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/random.py @@ -40,16 +40,16 @@ class HypothesisRandom(Random): return self.__copy__() def __repr__(self): - raise NotImplementedError() + raise NotImplementedError def seed(self, seed): - raise NotImplementedError() + raise NotImplementedError def getstate(self): - raise NotImplementedError() + raise NotImplementedError def setstate(self, state): - raise NotImplementedError() + raise NotImplementedError def _hypothesis_log_random(self, method, kwargs, result): if not (self.__note_method_calls and should_note()): @@ -62,7 +62,7 @@ class HypothesisRandom(Random): report(f"{self!r}.{method}({argstr}) -> {result!r}") def _hypothesis_do_random(self, method, kwargs): - raise NotImplementedError() + raise NotImplementedError RANDOM_METHODS = [ @@ -97,15 +97,15 @@ RANDOM_METHODS = [ # Fake shims to get a good signature def getrandbits(self, n: int) -> int: # type: ignore - raise NotImplementedError() + raise NotImplementedError def random(self) -> float: # type: ignore - raise NotImplementedError() + raise NotImplementedError def _randbelow(self, n: int) -> int: # type: ignore - raise NotImplementedError() + raise NotImplementedError STUBS = {f.__name__: f for f in [getrandbits, random, _randbelow]} @@ -212,7 +212,7 @@ class ArtificialRandom(HypothesisRandom): original = list(seq) for i, i2 in enumerate(result): seq[i] = original[i2] - return + return None return result def _hypothesis_do_random(self, method, kwargs): @@ -223,7 +223,7 @@ class ArtificialRandom(HypothesisRandom): elif method == "shuffle": key = (method, len(kwargs["x"])) else: - key = (method,) + tuple(sorted(kwargs)) + key = (method, *sorted(kwargs)) try: result, self.__state = self.__state.next_states[key] @@ -296,14 +296,17 @@ class ArtificialRandom(HypothesisRandom): f"Sample size {k} not in expected range 0 <= k <= {len(seq)}" ) - result = self.__data.draw( - lists( - sampled_from(range(len(seq))), - min_size=k, - max_size=k, - unique=True, + if k == 0: + result = [] + else: + result = self.__data.draw( + lists( + sampled_from(range(len(seq))), + min_size=k, + max_size=k, + unique=True, + ) ) - ) elif method == "getrandbits": result = self.__data.draw_bits(kwargs["n"]) @@ -388,7 +391,7 @@ def convert_kwargs(name, kwargs): if args[-1] is signature.parameters[name].default: args.pop() else: - break # pragma: no cover # Only on Python < 3.8 + break return (args, kwargs) diff --git a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/recursive.py b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/recursive.py index 58b218ab07..7709b45460 100644 --- a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/recursive.py +++ b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/recursive.py @@ -57,7 +57,7 @@ class LimitedStrategy(SearchStrategy): def do_draw(self, data): assert self.currently_capped if self.marker <= 0: - raise LimitReached() + raise LimitReached self.marker -= 1 return data.draw(self.base_strategy) diff --git a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/regex.py b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/regex.py index 2e6611377b..a7f99c4ae5 100644 --- a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/regex.py +++ b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/regex.py @@ -11,16 +11,27 @@ import operator import re +from hypothesis.errors import InvalidArgument +from hypothesis.internal import charmap +from hypothesis.strategies._internal.lazy import unwrap_strategies +from hypothesis.strategies._internal.strings import OneCharStringStrategy + try: # pragma: no cover import re._constants as sre import re._parser as sre_parse + + ATOMIC_GROUP = sre.ATOMIC_GROUP + POSSESSIVE_REPEAT = sre.POSSESSIVE_REPEAT except ImportError: # Python < 3.11 import sre_constants as sre import sre_parse + ATOMIC_GROUP = object() + POSSESSIVE_REPEAT = object() + from hypothesis import reject, strategies as st from hypothesis.internal.charmap import as_general_categories, categories -from hypothesis.internal.compat import int_to_byte +from hypothesis.internal.compat import add_note, int_to_byte UNICODE_CATEGORIES = set(categories()) @@ -84,6 +95,14 @@ def clear_cache_after_draw(draw, base_strategy): return result +def chars_not_in_alphabet(alphabet, string): + # Given a string, return a tuple of the characters which are not in alphabet + if alphabet is None: + return () + intset = unwrap_strategies(alphabet).intervals + return tuple(c for c in string if c not in intset) + + class Context: __slots__ = ["flags"] @@ -101,42 +120,38 @@ class CharactersBuilder: :param flags: Regex flags. They affect how and which characters are matched """ - def __init__(self, negate=False, flags=0): + def __init__(self, *, negate=False, flags=0, alphabet): self._categories = set() self._whitelist_chars = set() self._blacklist_chars = set() self._negate = negate self._ignorecase = flags & re.IGNORECASE - self._unicode = not bool(flags & re.ASCII) self.code_to_char = chr + self._alphabet = unwrap_strategies(alphabet) + if flags & re.ASCII: + self._alphabet = OneCharStringStrategy( + self._alphabet.intervals & charmap.query(max_codepoint=127) + ) @property def strategy(self): """Returns resulting strategy that generates configured char set.""" - max_codepoint = None if self._unicode else 127 - # Due to the .swapcase() issue described below (and in issue #2657), - # self._whitelist_chars may contain strings of len > 1. We therefore - # have some extra logic to filter them out of st.characters() args, - # but still generate them if allowed to. - if self._negate: - black_chars = self._blacklist_chars - self._whitelist_chars - return st.characters( - blacklist_categories=self._categories | {"Cc", "Cs"}, - blacklist_characters={c for c in self._whitelist_chars if len(c) == 1}, - whitelist_characters=black_chars, - max_codepoint=max_codepoint, - ) + # Start by getting the set of all characters allowed by the pattern white_chars = self._whitelist_chars - self._blacklist_chars multi_chars = {c for c in white_chars if len(c) > 1} - char_strategy = st.characters( - whitelist_categories=self._categories, - blacklist_characters=self._blacklist_chars, - whitelist_characters=white_chars - multi_chars, - max_codepoint=max_codepoint, + intervals = charmap.query( + categories=self._categories, + exclude_characters=self._blacklist_chars, + include_characters=white_chars - multi_chars, + ) + # Then take the complement if this is from a negated character class + if self._negate: + intervals = charmap.query() - intervals + multi_chars.clear() + # and finally return the intersection with our alphabet + return OneCharStringStrategy(intervals & self._alphabet.intervals) | ( + st.sampled_from(sorted(multi_chars)) if multi_chars else st.nothing() ) - if multi_chars: - char_strategy |= st.sampled_from(sorted(multi_chars)) - return char_strategy def add_category(self, category): """Update unicode state to match sre_parse object ``category``.""" @@ -146,14 +161,10 @@ class CharactersBuilder: self._categories |= UNICODE_CATEGORIES - UNICODE_DIGIT_CATEGORIES elif category == sre.CATEGORY_SPACE: self._categories |= UNICODE_SPACE_CATEGORIES - self._whitelist_chars |= ( - UNICODE_SPACE_CHARS if self._unicode else SPACE_CHARS - ) + self._whitelist_chars |= UNICODE_SPACE_CHARS elif category == sre.CATEGORY_NOT_SPACE: self._categories |= UNICODE_CATEGORIES - UNICODE_SPACE_CATEGORIES - self._blacklist_chars |= ( - UNICODE_SPACE_CHARS if self._unicode else SPACE_CHARS - ) + self._blacklist_chars |= UNICODE_SPACE_CHARS elif category == sre.CATEGORY_WORD: self._categories |= UNICODE_WORD_CATEGORIES self._whitelist_chars.add("_") @@ -163,9 +174,11 @@ class CharactersBuilder: else: raise NotImplementedError(f"Unknown character category: {category}") - def add_char(self, char): + def add_char(self, char, *, check=True): """Add given char to the whitelist.""" c = self.code_to_char(char) + if check and chars_not_in_alphabet(self._alphabet, c): + raise InvalidArgument(f"Literal {c!r} is not in the specified alphabet") self._whitelist_chars.add(c) if ( self._ignorecase @@ -176,10 +189,11 @@ class CharactersBuilder: class BytesBuilder(CharactersBuilder): - def __init__(self, negate=False, flags=0): + def __init__(self, *, negate=False, flags=0): self._whitelist_chars = set() self._blacklist_chars = set() self._negate = negate + self._alphabet = None self._ignorecase = flags & re.IGNORECASE self.code_to_char = int_to_byte @@ -210,15 +224,25 @@ def maybe_pad(draw, regex, strategy, left_pad_strategy, right_pad_strategy): return result -def base_regex_strategy(regex, parsed=None): +def base_regex_strategy(regex, parsed=None, alphabet=None): if parsed is None: parsed = sre_parse.parse(regex.pattern, flags=regex.flags) - return clear_cache_after_draw( - _strategy(parsed, Context(flags=regex.flags), isinstance(regex.pattern, str)) - ) + try: + s = _strategy( + parsed, + context=Context(flags=regex.flags), + is_unicode=isinstance(regex.pattern, str), + alphabet=alphabet, + ) + except Exception as err: + add_note(err, f"{alphabet=} {regex=}") + raise + return clear_cache_after_draw(s) -def regex_strategy(regex, fullmatch, *, _temp_jsonschema_hack_no_end_newline=False): +def regex_strategy( + regex, fullmatch, *, alphabet, _temp_jsonschema_hack_no_end_newline=False +): if not hasattr(regex, "pattern"): regex = re.compile(regex) @@ -229,16 +253,16 @@ def regex_strategy(regex, fullmatch, *, _temp_jsonschema_hack_no_end_newline=Fal if fullmatch: if not parsed: return st.just("" if is_unicode else b"") - return base_regex_strategy(regex, parsed).filter(regex.fullmatch) + return base_regex_strategy(regex, parsed, alphabet).filter(regex.fullmatch) if not parsed: if is_unicode: - return st.text() + return st.text(alphabet=alphabet) else: return st.binary() if is_unicode: - base_padding_strategy = st.text() + base_padding_strategy = st.text(alphabet=alphabet) empty = st.just("") newline = st.just("\n") else: @@ -277,12 +301,12 @@ def regex_strategy(regex, fullmatch, *, _temp_jsonschema_hack_no_end_newline=Fal else: left_pad = empty - base = base_regex_strategy(regex, parsed).filter(regex.search) + base = base_regex_strategy(regex, parsed, alphabet).filter(regex.search) return maybe_pad(regex, base, left_pad, right_pad) -def _strategy(codes, context, is_unicode): +def _strategy(codes, context, is_unicode, *, alphabet): """Convert SRE regex parse tree to strategy that generates strings matching that regex represented by that parse tree. @@ -311,7 +335,7 @@ def _strategy(codes, context, is_unicode): """ def recurse(codes): - return _strategy(codes, context, is_unicode) + return _strategy(codes, context, is_unicode, alphabet=alphabet) if is_unicode: empty = "" @@ -335,8 +359,13 @@ def _strategy(codes, context, is_unicode): j += 1 if i + 1 < j: - chars = (to_char(charcode) for _, charcode in codes[i:j]) - strategies.append(st.just(empty.join(chars))) + chars = empty.join(to_char(charcode) for _, charcode in codes[i:j]) + if invalid := chars_not_in_alphabet(alphabet, chars): + raise InvalidArgument( + f"Literal {chars!r} contains characters {invalid!r} " + f"which are not in the specified alphabet" + ) + strategies.append(st.just(chars)) i = j continue @@ -357,10 +386,13 @@ def _strategy(codes, context, is_unicode): if code == sre.LITERAL: # Regex 'a' (single char) c = to_char(value) + if chars_not_in_alphabet(alphabet, c): + raise InvalidArgument(f"Literal {c!r} is not in the specified alphabet") if ( context.flags & re.IGNORECASE and c != c.swapcase() and re.match(re.escape(c), c.swapcase(), re.IGNORECASE) is not None + and not chars_not_in_alphabet(alphabet, c.swapcase()) ): # We do the explicit check for swapped-case matching because # eg 'ß'.upper() == 'SS' and ignorecase doesn't match it. @@ -393,7 +425,10 @@ def _strategy(codes, context, is_unicode): stack.extend(set(char.swapcase()) - blacklist) if is_unicode: - return st.characters(blacklist_characters=blacklist) + return OneCharStringStrategy( + unwrap_strategies(alphabet).intervals + & charmap.query(exclude_characters=blacklist) + ) else: return binary_char.filter(lambda c: c not in blacklist) @@ -401,9 +436,11 @@ def _strategy(codes, context, is_unicode): # Regex '[abc0-9]' (set of characters) negate = value[0][0] == sre.NEGATE if is_unicode: - builder = CharactersBuilder(negate, context.flags) + builder = CharactersBuilder( + flags=context.flags, negate=negate, alphabet=alphabet + ) else: - builder = BytesBuilder(negate, context.flags) + builder = BytesBuilder(flags=context.flags, negate=negate) for charset_code, charset_value in value: if charset_code == sre.NEGATE: @@ -417,7 +454,7 @@ def _strategy(codes, context, is_unicode): # Regex '[a-z]' (char range) low, high = charset_value for char_code in range(low, high + 1): - builder.add_char(char_code) + builder.add_char(char_code, check=char_code in (low, high)) elif charset_code == sre.CATEGORY: # Regex '[\w]' (char category) builder.add_category(charset_value) @@ -430,9 +467,13 @@ def _strategy(codes, context, is_unicode): elif code == sre.ANY: # Regex '.' (any char) if is_unicode: + assert alphabet is not None if context.flags & re.DOTALL: - return st.characters() - return st.characters(blacklist_characters="\n") + return alphabet + return OneCharStringStrategy( + unwrap_strategies(alphabet).intervals + & charmap.query(exclude_characters="\n") + ) else: if context.flags & re.DOTALL: return binary_char @@ -449,7 +490,7 @@ def _strategy(codes, context, is_unicode): old_flags = context.flags context.flags = (context.flags | value[1]) & ~value[2] - strat = _strategy(value[-1], context, is_unicode) + strat = _strategy(value[-1], context, is_unicode, alphabet=alphabet) context.flags = old_flags @@ -474,7 +515,7 @@ def _strategy(codes, context, is_unicode): # Regex 'a|b|c' (branch) return st.one_of([recurse(branch) for branch in value[1]]) - elif code in [sre.MIN_REPEAT, sre.MAX_REPEAT]: + elif code in [sre.MIN_REPEAT, sre.MAX_REPEAT, POSSESSIVE_REPEAT]: # Regexes 'a?', 'a*', 'a+' and their non-greedy variants # (repeaters) at_least, at_most, subregex = value @@ -494,8 +535,12 @@ def _strategy(codes, context, is_unicode): recurse(value[1]), recurse(value[2]) if value[2] else st.just(empty), ) + elif code == ATOMIC_GROUP: # pragma: no cover # new in Python 3.11 + return _strategy(value, context, is_unicode, alphabet=alphabet) else: # Currently there are no known code points other than handled here. # This code is just future proofing - raise NotImplementedError(f"Unknown code point: {code!r}") + raise NotImplementedError( + f"Unknown code point: {code!r}. Please open an issue." + ) diff --git a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/strategies.py b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/strategies.py index 9acb3f9d4a..415c9092ee 100644 --- a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/strategies.py +++ b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/strategies.py @@ -15,6 +15,7 @@ from random import shuffle from typing import ( Any, Callable, + ClassVar, Dict, Generic, List, @@ -421,7 +422,7 @@ class SearchStrategy(Generic[Ex]): self.validate_called = False raise - LABELS: Dict[type, int] = {} + LABELS: ClassVar[Dict[type, int]] = {} @property def class_label(self): @@ -484,14 +485,14 @@ class SampledFromStrategy(SearchStrategy): return type(self)( self.elements, repr_=self.repr_, - transformations=self._transformations + (("map", pack),), + transformations=(*self._transformations, ("map", pack)), ) def filter(self, condition): return type(self)( self.elements, repr_=self.repr_, - transformations=self._transformations + (("filter", condition),), + transformations=(*self._transformations, ("filter", condition)), ) def __repr__(self): @@ -697,26 +698,26 @@ def one_of( ... -@overload # noqa: F811 +@overload def one_of(__a1: SearchStrategy[Ex]) -> SearchStrategy[Ex]: # pragma: no cover ... -@overload # noqa: F811 +@overload def one_of( __a1: SearchStrategy[Ex], __a2: SearchStrategy[T] ) -> SearchStrategy[Union[Ex, T]]: # pragma: no cover ... -@overload # noqa: F811 +@overload def one_of( __a1: SearchStrategy[Ex], __a2: SearchStrategy[T], __a3: SearchStrategy[T3] ) -> SearchStrategy[Union[Ex, T, T3]]: # pragma: no cover ... -@overload # noqa: F811 +@overload def one_of( __a1: SearchStrategy[Ex], __a2: SearchStrategy[T], @@ -726,7 +727,7 @@ def one_of( ... -@overload # noqa: F811 +@overload def one_of( __a1: SearchStrategy[Ex], __a2: SearchStrategy[T], @@ -737,7 +738,7 @@ def one_of( ... -@overload # noqa: F811 +@overload def one_of(*args: SearchStrategy[Any]) -> SearchStrategy[Any]: # pragma: no cover ... @@ -745,7 +746,7 @@ def one_of(*args: SearchStrategy[Any]) -> SearchStrategy[Any]: # pragma: no cov @defines_strategy(never_lazy=True) def one_of( *args: Union[Sequence[SearchStrategy[Any]], SearchStrategy[Any]] -) -> SearchStrategy[Any]: # noqa: F811 +) -> SearchStrategy[Any]: # Mypy workaround alert: Any is too loose above; the return parameter # should be the union of the input parameters. Unfortunately, Mypy <=0.600 # raises errors due to incompatible inputs instead. See #1270 for links. @@ -810,10 +811,7 @@ class MappedSearchStrategy(SearchStrategy[Ex]): def __repr__(self): if not hasattr(self, "_cached_repr"): - self._cached_repr = "{!r}.map({})".format( - self.mapped_strategy, - get_pretty_function_description(self.pack), - ) + self._cached_repr = f"{self.mapped_strategy!r}.map({get_pretty_function_description(self.pack)})" return self._cached_repr def do_validate(self): @@ -840,7 +838,7 @@ class MappedSearchStrategy(SearchStrategy[Ex]): return result except UnsatisfiedAssumption: data.stop_example(discard=True) - raise UnsatisfiedAssumption() + raise UnsatisfiedAssumption @property def branches(self) -> List[SearchStrategy[Ex]]: diff --git a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/strings.py b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/strings.py index e1dc1f4da5..22952b6fe3 100644 --- a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/strings.py +++ b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/strings.py @@ -9,63 +9,73 @@ # obtain one at https://mozilla.org/MPL/2.0/. import copy +import re import warnings +from functools import lru_cache from hypothesis.errors import HypothesisWarning, InvalidArgument from hypothesis.internal import charmap from hypothesis.internal.conjecture.utils import biased_coin, integer_range from hypothesis.internal.intervalsets import IntervalSet from hypothesis.strategies._internal.collections import ListStrategy +from hypothesis.strategies._internal.lazy import unwrap_strategies from hypothesis.strategies._internal.strategies import SearchStrategy class OneCharStringStrategy(SearchStrategy): """A strategy which generates single character strings of text type.""" - def __init__( - self, - whitelist_categories=None, - blacklist_categories=None, - blacklist_characters=None, + def __init__(self, intervals, force_repr=None): + assert isinstance(intervals, IntervalSet) + self.intervals = intervals + self._force_repr = force_repr + self.zero_point = self.intervals.index_above(ord("0")) + self.Z_point = min( + self.intervals.index_above(ord("Z")), len(self.intervals) - 1 + ) + + @classmethod + def from_characters_args( + cls, + *, + codec=None, min_codepoint=None, max_codepoint=None, - whitelist_characters=None, + categories=None, + exclude_characters=None, + include_characters=None, ): - assert set(whitelist_categories or ()).issubset(charmap.categories()) - assert set(blacklist_categories or ()).issubset(charmap.categories()) + assert set(categories or ()).issubset(charmap.categories()) intervals = charmap.query( - include_categories=whitelist_categories, - exclude_categories=blacklist_categories, min_codepoint=min_codepoint, max_codepoint=max_codepoint, - include_characters=whitelist_characters, - exclude_characters=blacklist_characters, + categories=categories, + exclude_characters=exclude_characters, + include_characters=include_characters, ) - self._arg_repr = ", ".join( + if codec is not None: + intervals &= charmap.intervals_from_codec(codec) + _arg_repr = ", ".join( f"{k}={v!r}" for k, v in [ - ("whitelist_categories", whitelist_categories), - ("blacklist_categories", blacklist_categories), - ("whitelist_characters", whitelist_characters), - ("blacklist_characters", blacklist_characters), + ("codec", codec), ("min_codepoint", min_codepoint), ("max_codepoint", max_codepoint), + ("categories", categories), + ("exclude_characters", exclude_characters), + ("include_characters", include_characters), ] - if not (v in (None, "") or (k == "blacklist_categories" and v == ("Cs",))) + if v not in (None, "", set(charmap.categories()) - {"Cs"}) ) if not intervals: raise InvalidArgument( "No characters are allowed to be generated by this " - f"combination of arguments: {self._arg_repr}" + f"combination of arguments: {_arg_repr}" ) - self.intervals = IntervalSet(intervals) - self.zero_point = self.intervals.index_above(ord("0")) - self.Z_point = min( - self.intervals.index_above(ord("Z")), len(self.intervals) - 1 - ) + return cls(intervals, force_repr=f"characters({_arg_repr})") def __repr__(self): - return f"characters({self._arg_repr})" + return self._force_repr or f"OneCharStringStrategy({self.intervals!r})" def do_draw(self, data): if len(self.intervals) > 256: @@ -120,7 +130,8 @@ class TextStrategy(ListStrategy): # See https://docs.python.org/3/library/stdtypes.html#string-methods # These methods always return Truthy values for any nonempty string. - _nonempty_filters = ListStrategy._nonempty_filters + ( + _nonempty_filters = ( + *ListStrategy._nonempty_filters, str, str.capitalize, str.casefold, @@ -158,7 +169,30 @@ class TextStrategy(ListStrategy): f"You applied str.{condition.__name__} as a filter, but this allows " f"all nonempty strings! Did you mean str.is{condition.__name__}?", HypothesisWarning, + stacklevel=2, ) + elems = unwrap_strategies(self.element_strategy) + if ( + condition is str.isidentifier + and self.max_size >= 1 + and isinstance(elems, OneCharStringStrategy) + ): + from hypothesis.strategies import builds, nothing + + id_start, id_continue = _identifier_characters() + if not (elems.intervals & id_start): + return nothing() + return builds( + "{}{}".format, + OneCharStringStrategy(elems.intervals & id_start), + TextStrategy( + OneCharStringStrategy(elems.intervals & id_continue), + min_size=max(0, self.min_size - 1), + max_size=self.max_size - 1, + ), + # Filter to ensure that NFKC normalization keeps working in future + ).filter(str.isidentifier) + # We use ListStrategy filter logic for the conditions that *only* imply # the string is nonempty. Here, we increment the min_size but still apply # the filter for conditions that imply nonempty *and specific contents*. @@ -171,6 +205,59 @@ class TextStrategy(ListStrategy): return super().filter(condition) +# Excerpted from https://www.unicode.org/Public/15.0.0/ucd/PropList.txt +# Python updates it's Unicode version between minor releases, but fortunately +# these properties do not change between the Unicode versions in question. +_PROPLIST = """ +# ================================================ + +1885..1886 ; Other_ID_Start # Mn [2] MONGOLIAN LETTER ALI GALI BALUDA..MONGOLIAN LETTER ALI GALI THREE BALUDA +2118 ; Other_ID_Start # Sm SCRIPT CAPITAL P +212E ; Other_ID_Start # So ESTIMATED SYMBOL +309B..309C ; Other_ID_Start # Sk [2] KATAKANA-HIRAGANA VOICED SOUND MARK..KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK + +# Total code points: 6 + +# ================================================ + +00B7 ; Other_ID_Continue # Po MIDDLE DOT +0387 ; Other_ID_Continue # Po GREEK ANO TELEIA +1369..1371 ; Other_ID_Continue # No [9] ETHIOPIC DIGIT ONE..ETHIOPIC DIGIT NINE +19DA ; Other_ID_Continue # No NEW TAI LUE THAM DIGIT ONE + +# Total code points: 12 +""" + + +@lru_cache +def _identifier_characters(): + """See https://docs.python.org/3/reference/lexical_analysis.html#identifiers""" + # Start by computing the set of special characters + chars = {"Other_ID_Start": "", "Other_ID_Continue": ""} + for line in _PROPLIST.splitlines(): + if m := re.match(r"([0-9A-F.]+) +; (\w+) # ", line): + codes, prop = m.groups() + span = range(int(codes[:4], base=16), int(codes[-4:], base=16) + 1) + chars[prop] += "".join(chr(x) for x in span) + + # Then get the basic set by Unicode category and known extras + id_start = charmap.query( + categories=("Lu", "Ll", "Lt", "Lm", "Lo", "Nl"), + include_characters="_" + chars["Other_ID_Start"], + ) + id_start -= IntervalSet.from_string( + # Magic value: the characters which NFKC-normalize to be invalid identifiers. + # Conveniently they're all in `id_start`, so we only need to do this once. + "\u037a\u0e33\u0eb3\u2e2f\u309b\u309c\ufc5e\ufc5f\ufc60\ufc61\ufc62\ufc63" + "\ufdfa\ufdfb\ufe70\ufe72\ufe74\ufe76\ufe78\ufe7a\ufe7c\ufe7e\uff9e\uff9f" + ) + id_continue = id_start | charmap.query( + categories=("Mn", "Mc", "Nd", "Pc"), + include_characters=chars["Other_ID_Continue"], + ) + return id_start, id_continue + + class FixedSizeBytes(SearchStrategy): def __init__(self, size): self.size = size diff --git a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/types.py b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/types.py index 5576c8f2d1..51185fb38f 100644 --- a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/types.py +++ b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/types.py @@ -10,6 +10,7 @@ import builtins import collections +import collections.abc import datetime import decimal import fractions @@ -19,21 +20,18 @@ import io import ipaddress import numbers import os +import random import re import sys import typing import uuid from pathlib import PurePath from types import FunctionType +from typing import get_args, get_origin from hypothesis import strategies as st from hypothesis.errors import InvalidArgument, ResolutionFailed -from hypothesis.internal.compat import ( - PYPY, - BaseExceptionGroup, - ExceptionGroup, - get_origin, -) +from hypothesis.internal.compat import PYPY, BaseExceptionGroup, ExceptionGroup from hypothesis.internal.conjecture.utils import many as conjecture_utils_many from hypothesis.strategies._internal.datetime import zoneinfo # type: ignore from hypothesis.strategies._internal.ipaddress import ( @@ -82,11 +80,7 @@ try: except AttributeError: # pragma: no cover pass # `typing_extensions` might not be installed -FinalTypes: tuple = () -try: - FinalTypes += (typing.Final,) -except AttributeError: # pragma: no cover - pass # Is missing for `python<3.8` +FinalTypes: tuple = (typing.Final,) try: FinalTypes += (typing_extensions.Final,) except AttributeError: # pragma: no cover @@ -96,7 +90,7 @@ ConcatenateTypes: tuple = () try: ConcatenateTypes += (typing.Concatenate,) except AttributeError: # pragma: no cover - pass # Is missing for `python<3.8` + pass # Is missing for `python<3.10` try: ConcatenateTypes += (typing_extensions.Concatenate,) except AttributeError: # pragma: no cover @@ -106,7 +100,7 @@ ParamSpecTypes: tuple = () try: ParamSpecTypes += (typing.ParamSpec,) except AttributeError: # pragma: no cover - pass # Is missing for `python<3.8` + pass # Is missing for `python<3.10` try: ParamSpecTypes += (typing_extensions.ParamSpec,) except AttributeError: # pragma: no cover @@ -116,7 +110,7 @@ TypeGuardTypes: tuple = () try: TypeGuardTypes += (typing.TypeGuard,) except AttributeError: # pragma: no cover - pass # Is missing for `python<3.8` + pass # Is missing for `python<3.10` try: TypeGuardTypes += (typing_extensions.TypeGuard,) except AttributeError: # pragma: no cover @@ -151,8 +145,7 @@ typing_root_type = (typing._Final, typing._GenericAlias) # type: ignore # We use this to disallow all non-runtime types from being registered and resolved. # By "non-runtime" we mean: types that do not really exist in python's # and are just added for more fancy type annotations. -# `Final` is a great example: it just indicates -# that this value can't be reassigned. +# `Final` is a great example: it just indicates that this value can't be reassigned. NON_RUNTIME_TYPES = ( typing.Any, *ClassVarTypes, @@ -185,15 +178,15 @@ for name in ( def type_sorting_key(t): """Minimise to None, then non-container types, then container types.""" - if not (is_a_type(t) or is_typing_literal(t)): # This branch is for Python < 3.8 - raise InvalidArgument(f"thing={t} must be a type") # pragma: no cover - if t is None or t is type(None): # noqa: E721 + if t is None or t is type(None): return (-1, repr(t)) - t = getattr(t, "__origin__", t) - if not isinstance(t, type): # pragma: no cover - # Some generics in the typing module are not actually types in 3.7 - return (2, repr(t)) - return (int(issubclass(t, collections.abc.Container)), repr(t)) + t = get_origin(t) or t + try: + is_container = int(issubclass(t, collections.abc.Container)) + except Exception: # pragma: no cover + # e.g. `typing_extensions.Literal` is not a container + is_container = 0 + return (is_container, repr(t)) def _compatible_args(args, superclass_args): @@ -214,11 +207,8 @@ def try_issubclass(thing, superclass): try: # In this case we're looking at two distinct classes - which might be generics. # That brings in some complications: - if issubclass( - getattr(thing, "__origin__", None) or thing, - getattr(superclass, "__origin__", None) or superclass, - ): - superclass_args = getattr(superclass, "__args__", None) + if issubclass(get_origin(thing) or thing, get_origin(superclass) or superclass): + superclass_args = get_args(superclass) if not superclass_args: # The superclass is not generic, so we're definitely a subclass. return True @@ -250,15 +240,12 @@ def is_a_new_type(thing): ) # In 3.10 and later, NewType is actually a class - which simplifies things. # See https://bugs.python.org/issue44353 for links to the various patches. - return isinstance(thing, typing.NewType) # pragma: no cover # on 3.8, anyway + return isinstance(thing, typing.NewType) def is_a_union(thing): """Return True if thing is a typing.Union or types.UnionType (in py310).""" - return ( - isinstance(thing, UnionType) - or getattr(thing, "__origin__", None) is typing.Union - ) + return isinstance(thing, UnionType) or get_origin(thing) is typing.Union def is_a_type(thing): @@ -267,11 +254,9 @@ def is_a_type(thing): def is_typing_literal(thing): - return ( - hasattr(typing, "Literal") - and getattr(thing, "__origin__", None) == typing.Literal - or hasattr(typing_extensions, "Literal") - and getattr(thing, "__origin__", None) == typing_extensions.Literal + return get_origin(thing) in ( + typing.Literal, + getattr(typing_extensions, "Literal", object()), ) @@ -314,8 +299,9 @@ def is_generic_type(type_): # The ugly truth is that `MyClass`, `MyClass[T]`, and `MyClass[int]` are very different. # We check for `MyClass[T]` and `MyClass[int]` with the first condition, # while the second condition is for `MyClass`. - return isinstance(type_, typing_root_type + (GenericAlias,)) or ( - isinstance(type_, type) and typing.Generic in type_.__mro__ + return isinstance(type_, (*typing_root_type, GenericAlias)) or ( + isinstance(type_, type) + and (typing.Generic in type_.__mro__ or hasattr(type_, "__class_getitem__")) ) @@ -345,7 +331,7 @@ def from_typing_type(thing): # information to sensibly resolve to strategies at runtime. # Finally, we run a variation of the subclass lookup in `st.from_type` # among generic types in the lookup. - if getattr(thing, "__origin__", None) == tuple or isinstance( + if get_origin(thing) == tuple or isinstance( thing, getattr(typing, "TupleMeta", ()) ): elem_types = getattr(thing, "__tuple_params__", None) or () @@ -359,7 +345,7 @@ def from_typing_type(thing): elif len(elem_types) == 1 and elem_types[0] == (): return st.tuples() # Empty tuple; see issue #1583 return st.tuples(*map(st.from_type, elem_types)) - if hasattr(typing, "Final") and getattr(thing, "__origin__", None) == typing.Final: + if get_origin(thing) == typing.Final: return st.one_of([st.from_type(t) for t in thing.__args__]) if is_typing_literal(thing): args_dfs_stack = list(thing.__args__) @@ -388,8 +374,8 @@ def from_typing_type(thing): raise ResolutionFailed(f"Cannot resolve {thing} to a strategy") # Some "generic" classes are not generic *in* anything - for example both - # Hashable and Sized have `__args__ == ()` on Python 3.7 or later. - origin = getattr(thing, "__origin__", thing) + # Hashable and Sized have `__args__ == ()` + origin = get_origin(thing) or thing if ( origin in vars(collections.abc).values() and len(getattr(thing, "__args__", None) or []) == 0 @@ -399,19 +385,35 @@ def from_typing_type(thing): # Parametrised generic types have their __origin__ attribute set to the # un-parametrised version, which we need to use in the subclass checks. # e.g.: typing.List[int].__origin__ == typing.List + # (actually not sure if this is true since Python 3.9 or so) mapping = { k: v for k, v in _global_type_lookup.items() if is_generic_type(k) and try_issubclass(k, thing) } - if typing.Dict in mapping or typing.Set in mapping: + # Drop some unusual cases for simplicity, including tuples or its + # subclasses (e.g. namedtuple) + if len(mapping) > 1: + _Environ = getattr(os, "_Environ", None) + mapping.pop(_Environ, None) + tuple_types = [t for t in mapping if isinstance(t, type) and issubclass(t, tuple)] + if len(mapping) > len(tuple_types): + for tuple_type in tuple_types: + mapping.pop(tuple_type) + + # After we drop Python 3.8 and can rely on having generic builtin types, we'll + # be able to simplify this logic by dropping the typing-module handling. + if {dict, set, typing.Dict, typing.Set}.intersection(mapping): # ItemsView can cause test_lookup.py::test_specialised_collection_types # to fail, due to weird isinstance behaviour around the elements. + mapping.pop(collections.abc.ItemsView, None) mapping.pop(typing.ItemsView, None) - if typing.Deque in mapping and len(mapping) > 1: + if {collections.deque, typing.Deque}.intersection(mapping) and len(mapping) > 1: # Resolving generic sequences to include a deque is more trouble for e.g. # the ghostwriter than it's worth, via undefined names in the repr. - mapping.pop(typing.Deque) + mapping.pop(collections.deque, None) + mapping.pop(typing.Deque, None) + if len(mapping) > 1: # issubclass treats bytestring as a kind of sequence, which it is, # but treating it as such breaks everything else when it is presumed @@ -419,7 +421,7 @@ def from_typing_type(thing): # Except for sequences of integers, or unions which include integer! # See https://github.com/HypothesisWorks/hypothesis/issues/2257 # - # This block drops ByteString from the types that can be generated + # This block drops bytes from the types that can be generated # if there is more than one allowed type, and the element type is # not either `int` or a Union with `int` as one of its elements. elem_type = (getattr(thing, "__args__", None) or ["not int"])[0] @@ -429,8 +431,10 @@ def from_typing_type(thing): union_elems = () if not any( isinstance(T, type) and issubclass(int, get_origin(T) or T) - for T in list(union_elems) + [elem_type] + for T in [*union_elems, elem_type] ): + mapping.pop(bytes, None) + mapping.pop(collections.abc.ByteString, None) mapping.pop(typing.ByteString, None) elif ( (not mapping) @@ -438,10 +442,22 @@ def from_typing_type(thing): and thing.__forward_arg__ in vars(builtins) ): return st.from_type(getattr(builtins, thing.__forward_arg__)) + # Before Python 3.9, we sometimes have e.g. Sequence from both the typing + # module, and collections.abc module. Discard any type which is not it's own + # origin, where the origin is also in the mapping. + for t in sorted(mapping, key=type_sorting_key): + origin = get_origin(t) + if origin is not t and origin in mapping: + mapping.pop(t) + # Sort strategies according to our type-sorting heuristic for stable output strategies = [ - v if isinstance(v, st.SearchStrategy) else v(thing) - for k, v in mapping.items() - if sum(try_issubclass(k, T) for T in mapping) == 1 + s + for s in ( + v if isinstance(v, st.SearchStrategy) else v(thing) + for k, v in sorted(mapping.items(), key=lambda kv: type_sorting_key(kv[0])) + if sum(try_issubclass(k, T) for T in mapping) == 1 + ) + if s != NotImplemented ] empty = ", ".join(repr(s) for s in strategies if s.is_empty) if empty or not strategies: @@ -479,6 +495,14 @@ utc_offsets = st.builds( # As a general rule, we try to limit this to scalars because from_type() # would have to decide on arbitrary collection elements, and we'd rather # not (with typing module generic types and some builtins as exceptions). +# +# Strategy Callables may return NotImplemented, which should be treated in the +# same way as if the type was not registered. +# +# Note that NotImplemented cannot be typed in Python 3.8 because there's no type +# exposed for it, and NotImplemented itself is typed as Any so that it can be +# returned without being listed in a function signature: +# https://github.com/python/mypy/issues/6710#issuecomment-485580032 _global_type_lookup: typing.Dict[ type, typing.Union[st.SearchStrategy, typing.Callable[[type], st.SearchStrategy]] ] = { @@ -577,6 +601,7 @@ _global_type_lookup: typing.Dict[ super: st.builds(super, st.from_type(type)), re.Match: st.text().map(lambda c: re.match(".", c, flags=re.DOTALL)).filter(bool), re.Pattern: st.builds(re.compile, st.sampled_from(["", b""])), + random.Random: st.randoms(), # Pull requests with more types welcome! } if zoneinfo is not None: # pragma: no branch @@ -586,7 +611,7 @@ if PYPY: _global_type_lookup[type] = st.sampled_from( - [type(None)] + sorted(_global_type_lookup, key=str) + [type(None), *sorted(_global_type_lookup, key=str)] ) if sys.version_info[:2] >= (3, 9): # subclass of MutableMapping, and in Python 3.9 we resolve to a union @@ -598,9 +623,9 @@ _global_type_lookup.update( { # Note: while ByteString notionally also represents the bytearray and # memoryview types, it is a subclass of Hashable and those types are not. - # We therefore only generate the bytes type. - typing.ByteString: st.binary(), - collections.abc.ByteString: st.binary(), + # We therefore only generate the bytes type. type-ignored due to deprecation. + typing.ByteString: st.binary(), # type: ignore + collections.abc.ByteString: st.binary(), # type: ignore # TODO: SupportsAbs and SupportsRound should be covariant, ie have functions. typing.SupportsAbs: st.one_of( st.booleans(), @@ -641,6 +666,7 @@ _global_type_lookup.update( # this generates strings that should able to be parsed into integers st.from_regex(r"\A-?\d+\Z").filter(functools.partial(can_cast, int)), ), + typing.SupportsIndex: st.integers() | st.booleans(), typing.SupportsBytes: st.one_of( st.booleans(), st.binary(), @@ -652,8 +678,6 @@ _global_type_lookup.update( typing.TextIO: st.builds(io.StringIO, st.text()), } ) -if hasattr(typing, "SupportsIndex"): # pragma: no branch # new in Python 3.8 - _global_type_lookup[typing.SupportsIndex] = st.integers() | st.booleans() # The "extra" lookups define a callable that either resolves to a strategy for @@ -685,17 +709,23 @@ def register(type_, fallback=None, *, module=typing): return lambda f: f def inner(func): + nonlocal type_ if fallback is None: _global_type_lookup[type_] = func return func @functools.wraps(func) def really_inner(thing): - # This branch is for Python < 3.8, when __args__ was not always tracked + # This branch is for Python <= 3.8, when __args__ was not always tracked if getattr(thing, "__args__", None) is None: return fallback # pragma: no cover return func(thing) + if sys.version_info[:2] >= (3, 9): + try: + type_ = get_origin(type_) + except Exception: + pass _global_type_lookup[type_] = really_inner return really_inner @@ -706,7 +736,7 @@ def register(type_, fallback=None, *, module=typing): @register("Type", module=typing_extensions) def resolve_Type(thing): if getattr(thing, "__args__", None) is None: - # This branch is for Python < 3.8, when __args__ was not always tracked + # This branch is for Python <= 3.8, when __args__ was not always tracked return st.just(type) # pragma: no cover args = (thing.__args__[0],) if is_a_union(args[0]): @@ -755,6 +785,7 @@ def _from_hashable_type(type_): @register(typing.Set, st.builds(set)) +@register(typing.MutableSet, st.builds(set)) def resolve_Set(thing): return st.sets(_from_hashable_type(thing.__args__[0])) @@ -821,9 +852,8 @@ def resolve_ChainMap(thing): return resolve_Dict(thing).map(collections.ChainMap) -@register("OrderedDict", st.builds(dict).map(collections.OrderedDict)) +@register(typing.OrderedDict, st.builds(dict).map(collections.OrderedDict)) def resolve_OrderedDict(thing): - # typing.OrderedDict is new in Python 3.7.2 return resolve_Dict(thing).map(collections.OrderedDict) @@ -897,7 +927,7 @@ def resolve_Callable(thing): f"Callable type parametrized by {arg!r}. Consider using an " "explicit strategy, or opening an issue." ) - if getattr(return_type, "__origin__", None) in TypeGuardTypes: + if get_origin(return_type) in TypeGuardTypes: raise InvalidArgument( "Hypothesis cannot yet construct a strategy for callables which " f"are PEP-647 TypeGuards (got {return_type!r}). " diff --git a/contrib/python/hypothesis/py3/hypothesis/vendor/pretty.py b/contrib/python/hypothesis/py3/hypothesis/vendor/pretty.py index 3b8a5847c6..5a1989182a 100644 --- a/contrib/python/hypothesis/py3/hypothesis/vendor/pretty.py +++ b/contrib/python/hypothesis/py3/hypothesis/vendor/pretty.py @@ -65,10 +65,12 @@ Inheritance diagram: import datetime import re import struct +import sys import types import warnings from collections import defaultdict, deque from contextlib import contextmanager +from enum import Flag from io import StringIO from math import copysign, isnan @@ -79,9 +81,6 @@ __all__ = [ ] -_re_pattern_type = type(re.compile("")) - - def _safe_getattr(obj, attr, default=None): """Safe version of getattr. @@ -146,9 +145,19 @@ class RepresentationPrinter: self.snans = 0 self.stack = [] - self.singleton_pprinters = _singleton_pprinters.copy() - self.type_pprinters = _type_pprinters.copy() - self.deferred_pprinters = _deferred_type_pprinters.copy() + self.singleton_pprinters = {} + self.type_pprinters = {} + self.deferred_pprinters = {} + # If IPython has been imported, load up their pretty-printer registry + if "IPython.lib.pretty" in sys.modules: + ipp = sys.modules["IPython.lib.pretty"] + self.singleton_pprinters.update(ipp._singleton_pprinters) + self.type_pprinters.update(ipp._type_pprinters) + self.deferred_pprinters.update(ipp._deferred_type_pprinters) + # If there's overlap between our pprinters and IPython's, we'll use ours. + self.singleton_pprinters.update(_singleton_pprinters) + self.type_pprinters.update(_type_pprinters) + self.deferred_pprinters.update(_deferred_type_pprinters) # for which-parts-matter, we track a mapping from the (start_idx, end_idx) # of slices into the minimal failing example; this is per-interesting_origin @@ -308,21 +317,33 @@ class RepresentationPrinter: (usually the width of the opening text), the second and third the opening and closing delimiters. """ + self.begin_group(indent=indent, open=open) + try: + yield + finally: + self.end_group(dedent=indent, close=close) + + def begin_group(self, indent=0, open=""): + """Use the `with group(...) context manager instead. + + The begin_group() and end_group() methods are for IPython compatibility only; + see https://github.com/HypothesisWorks/hypothesis/issues/3721 for details. + """ if open: self.text(open) group = Group(self.group_stack[-1].depth + 1) self.group_stack.append(group) self.group_queue.enq(group) self.indentation += indent - try: - yield - finally: - self.indentation -= indent - group = self.group_stack.pop() - if not group.breakables: - self.group_queue.remove(group) - if close: - self.text(close) + + def end_group(self, dedent=0, close=""): + """See begin_group().""" + self.indentation -= dedent + group = self.group_stack.pop() + if not group.breakables: + self.group_queue.remove(group) + if close: + self.text(close) def _enumerate(self, seq): """Like enumerate, but with an upper limit on the number of items.""" @@ -419,7 +440,7 @@ class RepresentationPrinter: class Printable: def output(self, stream, output_width): # pragma: no cover - raise NotImplementedError() + raise NotImplementedError class Text(Printable): @@ -722,7 +743,7 @@ _type_pprinters = { set: _set_pprinter_factory("{", "}", set), frozenset: _set_pprinter_factory("frozenset({", "})", frozenset), super: _super_pprint, - _re_pattern_type: _re_pattern_pprint, + re.Pattern: _re_pattern_pprint, type: _type_pprint, types.FunctionType: _function_pprint, types.BuiltinFunctionType: _function_pprint, @@ -801,7 +822,14 @@ def _repr_dataframe(obj, p, cycle): # pragma: no cover def _repr_enum(obj, p, cycle): - p.text(type(obj).__name__ + "." + obj.name) + tname = type(obj).__name__ + if isinstance(obj, Flag): + p.text( + " | ".join(f"{tname}.{x.name}" for x in type(obj) if x & obj == x) + or f"{tname}({obj.value!r})" # if no matching members + ) + else: + p.text(f"{tname}.{obj.name}") for_type_by_name("collections", "defaultdict", _defaultdict_pprint) diff --git a/contrib/python/hypothesis/py3/hypothesis/vendor/tlds-alpha-by-domain.txt b/contrib/python/hypothesis/py3/hypothesis/vendor/tlds-alpha-by-domain.txt index c844d0d6bc..53f55386fa 100644 --- a/contrib/python/hypothesis/py3/hypothesis/vendor/tlds-alpha-by-domain.txt +++ b/contrib/python/hypothesis/py3/hypothesis/vendor/tlds-alpha-by-domain.txt @@ -1,4 +1,4 @@ -# Version 2023061100, Last Updated Sun Jun 11 07:07:01 2023 UTC +# Version 2023081200, Last Updated Sat Aug 12 07:07:01 2023 UTC AAA AARP ABB @@ -276,7 +276,6 @@ CONSULTING CONTACT CONTRACTORS COOKING -COOKINGCHANNEL COOL COOP CORSICA @@ -433,7 +432,6 @@ FM FO FOO FOOD -FOODNETWORK FOOTBALL FORD FOREX @@ -539,7 +537,6 @@ HELP HELSINKI HERE HERMES -HGTV HIPHOP HISAMITSU HITACHI @@ -561,7 +558,6 @@ HOSPITAL HOST HOSTING HOT -HOTELES HOTELS HOTMAIL HOUSE @@ -806,7 +802,6 @@ MTR MU MUSEUM MUSIC -MUTUAL MV MW MX @@ -847,7 +842,6 @@ NISSAY NL NO NOKIA -NORTHWESTERNMUTUAL NORTON NOW NOWRUZ @@ -893,7 +887,6 @@ PARS PARTNERS PARTS PARTY -PASSAGENS PAY PCCW PE @@ -1165,7 +1158,6 @@ THEATRE TIAA TICKETS TIENDA -TIFFANY TIPS TIRES TIROL @@ -1195,7 +1187,6 @@ TRADE TRADING TRAINING TRAVEL -TRAVELCHANNEL TRAVELERS TRAVELERSINSURANCE TRUST @@ -1257,7 +1248,6 @@ VOTING VOTO VOYAGE VU -VUELOS WALES WALMART WALTER diff --git a/contrib/python/hypothesis/py3/hypothesis/version.py b/contrib/python/hypothesis/py3/hypothesis/version.py index a97e4a8bb8..a28ac7bd63 100644 --- a/contrib/python/hypothesis/py3/hypothesis/version.py +++ b/contrib/python/hypothesis/py3/hypothesis/version.py @@ -8,5 +8,5 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -__version_info__ = (6, 78, 1) +__version_info__ = (6, 88, 1) __version__ = ".".join(map(str, __version_info__)) diff --git a/contrib/python/hypothesis/py3/ya.make b/contrib/python/hypothesis/py3/ya.make index 85b612db9a..ba30680ab2 100644 --- a/contrib/python/hypothesis/py3/ya.make +++ b/contrib/python/hypothesis/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(6.78.1) +VERSION(6.88.1) LICENSE(MPL-2.0) @@ -30,7 +30,6 @@ PY_SRCS( hypothesis/database.py hypothesis/entry_points.py hypothesis/errors.py - hypothesis/executors.py hypothesis/extra/__init__.py hypothesis/extra/_array_helpers.py hypothesis/extra/_patching.py |