diff options
author | Alexander Smirnov <alex@ydb.tech> | 2024-09-26 14:44:45 +0000 |
---|---|---|
committer | Alexander Smirnov <alex@ydb.tech> | 2024-09-26 14:44:45 +0000 |
commit | c853e78c5e416ede1d99665049657f2e304872e4 (patch) | |
tree | b6e2e59629b8e0f863eec9a700121ca3f7416f28 /contrib/python | |
parent | 7d208c76420539a7e44f2393c66e7d0444744517 (diff) | |
parent | 47e779420bb80722978c1ffd518245bffd886a50 (diff) | |
download | ydb-c853e78c5e416ede1d99665049657f2e304872e4.tar.gz |
Merge branch 'rightlib' into mergelibs-240926-1443
Diffstat (limited to 'contrib/python')
35 files changed, 350 insertions, 150 deletions
diff --git a/contrib/python/cffi/py3/.dist-info/METADATA b/contrib/python/cffi/py3/.dist-info/METADATA index a0d90a31e9..60b0779f68 100644 --- a/contrib/python/cffi/py3/.dist-info/METADATA +++ b/contrib/python/cffi/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: cffi -Version: 1.17.0 +Version: 1.17.1 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/contrib/python/cffi/py3/c/_cffi_backend.c b/contrib/python/cffi/py3/c/_cffi_backend.c index aa4ec48ba9..4ba773a175 100644 --- a/contrib/python/cffi/py3/c/_cffi_backend.c +++ b/contrib/python/cffi/py3/c/_cffi_backend.c @@ -2,7 +2,7 @@ #include <Python.h> #include "structmember.h" -#define CFFI_VERSION "1.17.0" +#define CFFI_VERSION "1.17.1" #ifdef MS_WIN32 #include <windows.h> diff --git a/contrib/python/cffi/py3/cffi/__init__.py b/contrib/python/cffi/py3/cffi/__init__.py index deeacc57cd..2e35a38c9c 100644 --- a/contrib/python/cffi/py3/cffi/__init__.py +++ b/contrib/python/cffi/py3/cffi/__init__.py @@ -5,8 +5,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing from .error import PkgConfigError -__version__ = "1.17.0" -__version_info__ = (1, 17, 0) +__version__ = "1.17.1" +__version_info__ = (1, 17, 1) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/contrib/python/cffi/py3/cffi/_embedding.h b/contrib/python/cffi/py3/cffi/_embedding.h index 0eeeea831d..94d8b30a9e 100644 --- a/contrib/python/cffi/py3/cffi/_embedding.h +++ b/contrib/python/cffi/py3/cffi/_embedding.h @@ -225,7 +225,7 @@ static int _cffi_initialize_python(void) if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.17.0" + "\ncompiled with cffi version: 1.17.1" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/contrib/python/cffi/py3/cffi/_shimmed_dist_utils.py b/contrib/python/cffi/py3/cffi/_shimmed_dist_utils.py index 611bf40f40..c3d2312818 100644 --- a/contrib/python/cffi/py3/cffi/_shimmed_dist_utils.py +++ b/contrib/python/cffi/py3/cffi/_shimmed_dist_utils.py @@ -30,7 +30,11 @@ try: from distutils.log import set_threshold, set_verbosity if sys.platform == 'win32': - from distutils.msvc9compiler import MSVCCompiler + try: + # FUTURE: msvc9compiler module was removed in setuptools 74; consider removing, as it's only used by an ancient patch in `recompiler` + from distutils.msvc9compiler import MSVCCompiler + except ImportError: + MSVCCompiler = None except Exception as ex: if sys.version_info >= (3, 12): raise Exception("This CFFI feature requires setuptools on Python >= 3.12. Please install the setuptools package.") from ex diff --git a/contrib/python/cffi/py3/cffi/api.py b/contrib/python/cffi/py3/cffi/api.py index edeb792810..5a474f3da9 100644 --- a/contrib/python/cffi/py3/cffi/api.py +++ b/contrib/python/cffi/py3/cffi/api.py @@ -693,7 +693,8 @@ class FFI(object): raise TypeError("emit_c_code() is only for C extension modules, " "not for dlopen()-style pure Python modules") recompile(self, module_name, source, - c_file=filename, call_c_compiler=False, **kwds) + c_file=filename, call_c_compiler=False, + uses_ffiplatform=False, **kwds) def emit_python_code(self, filename): from .recompiler import recompile @@ -705,7 +706,8 @@ class FFI(object): raise TypeError("emit_python_code() is only for dlopen()-style " "pure Python modules, not for C extension modules") recompile(self, module_name, source, - c_file=filename, call_c_compiler=False, **kwds) + c_file=filename, call_c_compiler=False, + uses_ffiplatform=False, **kwds) def compile(self, tmpdir='.', verbose=0, target=None, debug=None): """The 'target' argument gives the final file name of the diff --git a/contrib/python/cffi/py3/cffi/recompiler.py b/contrib/python/cffi/py3/cffi/recompiler.py index dd22b21c2c..c236807fe8 100644 --- a/contrib/python/cffi/py3/cffi/recompiler.py +++ b/contrib/python/cffi/py3/cffi/recompiler.py @@ -1417,6 +1417,10 @@ else: s = s.encode('ascii') super(NativeIO, self).write(s) +def _is_file_like(maybefile): + # compare to xml.etree.ElementTree._get_writer + return hasattr(maybefile, 'write') + def _make_c_or_py_source(ffi, module_name, preamble, target_file, verbose): if verbose: print("generating %s" % (target_file,)) @@ -1424,6 +1428,9 @@ def _make_c_or_py_source(ffi, module_name, preamble, target_file, verbose): target_is_python=(preamble is None)) recompiler.collect_type_table() recompiler.collect_step_tables() + if _is_file_like(target_file): + recompiler.write_source_to_f(target_file, preamble) + return True f = NativeIO() recompiler.write_source_to_f(f, preamble) output = f.getvalue() @@ -1481,9 +1488,12 @@ def _unpatch_meths(patchlist): def _patch_for_embedding(patchlist): if sys.platform == 'win32': # we must not remove the manifest when building for embedding! + # FUTURE: this module was removed in setuptools 74; this is likely dead code and should be removed, + # since the toolchain it supports (VS2005-2008) is also long dead. from cffi._shimmed_dist_utils import MSVCCompiler - _patch_meth(patchlist, MSVCCompiler, '_remove_visual_c_ref', - lambda self, manifest_file: manifest_file) + if MSVCCompiler is not None: + _patch_meth(patchlist, MSVCCompiler, '_remove_visual_c_ref', + lambda self, manifest_file: manifest_file) if sys.platform == 'darwin': # we must not make a '-bundle', but a '-dynamiclib' instead @@ -1517,12 +1527,16 @@ def _patch_for_target(patchlist, target): def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True, c_file=None, source_extension='.c', extradir=None, - compiler_verbose=1, target=None, debug=None, **kwds): + compiler_verbose=1, target=None, debug=None, + uses_ffiplatform=True, **kwds): if not isinstance(module_name, str): module_name = module_name.encode('ascii') if ffi._windows_unicode: ffi._apply_windows_unicode(kwds) if preamble is not None: + if call_c_compiler and _is_file_like(c_file): + raise TypeError("Writing to file-like objects is not supported " + "with call_c_compiler=True") embedding = (ffi._embedding is not None) if embedding: ffi._apply_embedding_fix(kwds) @@ -1541,7 +1555,10 @@ def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True, else: target = '*' # - ext = ffiplatform.get_extension(ext_c_file, module_name, **kwds) + if uses_ffiplatform: + ext = ffiplatform.get_extension(ext_c_file, module_name, **kwds) + else: + ext = None updated = make_c_source(ffi, module_name, preamble, c_file, verbose=compiler_verbose) if call_c_compiler: diff --git a/contrib/python/cffi/py3/ya.make b/contrib/python/cffi/py3/ya.make index 30e438d17e..90ddf448de 100644 --- a/contrib/python/cffi/py3/ya.make +++ b/contrib/python/cffi/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(1.17.0) +VERSION(1.17.1) LICENSE(MIT) diff --git a/contrib/python/executing/.dist-info/METADATA b/contrib/python/executing/.dist-info/METADATA index b598e4907d..45ff9aa881 100644 --- a/contrib/python/executing/.dist-info/METADATA +++ b/contrib/python/executing/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: executing -Version: 2.0.1 +Version: 2.1.0 Summary: Get the currently executing AST node of a frame, and other information Home-page: https://github.com/alexmojaki/executing Author: Alex Hall @@ -9,25 +9,23 @@ License: MIT Classifier: License :: OSI Approved :: MIT License Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.5 -Classifier: Programming Language :: Python :: 3.6 -Classifier: Programming Language :: Python :: 3.7 Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 -Requires-Python: >=3.5 +Classifier: Programming Language :: Python :: 3.13 +Requires-Python: >=3.8 Description-Content-Type: text/markdown License-File: LICENSE.txt Provides-Extra: tests -Requires-Dist: asttokens >=2.1.0 ; extra == 'tests' -Requires-Dist: ipython ; extra == 'tests' -Requires-Dist: pytest ; extra == 'tests' -Requires-Dist: coverage ; extra == 'tests' -Requires-Dist: coverage-enable-subprocess ; extra == 'tests' -Requires-Dist: littleutils ; extra == 'tests' -Requires-Dist: rich ; (python_version >= "3.11") and extra == 'tests' +Requires-Dist: asttokens>=2.1.0; extra == "tests" +Requires-Dist: ipython; extra == "tests" +Requires-Dist: pytest; extra == "tests" +Requires-Dist: coverage; extra == "tests" +Requires-Dist: coverage-enable-subprocess; extra == "tests" +Requires-Dist: littleutils; extra == "tests" +Requires-Dist: rich; python_version >= "3.11" and extra == "tests" # executing diff --git a/contrib/python/executing/executing/_position_node_finder.py b/contrib/python/executing/executing/_position_node_finder.py index 8ca21a67bd..7a814150da 100644 --- a/contrib/python/executing/executing/_position_node_finder.py +++ b/contrib/python/executing/executing/_position_node_finder.py @@ -72,7 +72,7 @@ def mangled_name(node: EnhancedAST) -> str: @lru_cache(128) # pragma: no mutate def get_instructions(code: CodeType) -> list[dis.Instruction]: - return list(dis.get_instructions(code, show_caches=True)) + return list(dis.get_instructions(code)) types_cmp_issue_fix = ( @@ -114,7 +114,7 @@ class PositionNodeFinder(object): """ def __init__(self, frame: FrameType, stmts: Set[EnhancedAST], tree: ast.Module, lasti: int, source: Source): - self.bc_list = get_instructions(frame.f_code) + self.bc_dict={bc.offset:bc for bc in get_instructions(frame.f_code) } self.source = source self.decorator: Optional[EnhancedAST] = None @@ -141,7 +141,7 @@ class PositionNodeFinder(object): # we ignore here the start position and try to find the ast-node just by end position and expected node type # This is save, because there can only be one attribute ending at a specific point in the source code. typ = (ast.Attribute,) - elif self.opname(lasti) == "CALL": + elif self.opname(lasti) in ("CALL", "CALL_KW"): # A CALL instruction can be a method call, in which case the lineno and col_offset gets changed by the compiler. # Therefore we ignoring here this attributes and searchnig for a Call-node only by end_col_offset and end_lineno. # This is save, because there can only be one method ending at a specific point in the source code. @@ -156,13 +156,18 @@ class PositionNodeFinder(object): typ=typ, ) - self.known_issues(self.result, self.instruction(lasti)) + instruction = self.instruction(lasti) + assert instruction is not None + + self.result = self.fix_result(self.result, instruction) + + self.known_issues(self.result, instruction) self.test_for_decorator(self.result, lasti) # verify if self.decorator is None: - self.verify(self.result, self.instruction(lasti)) + self.verify(self.result, instruction) else: assert_(self.decorator in self.result.decorator_list) @@ -213,6 +218,32 @@ class PositionNodeFinder(object): if sys.version_info < (3, 12): index += 4 + def fix_result( + self, node: EnhancedAST, instruction: dis.Instruction + ) -> EnhancedAST: + if ( + sys.version_info >= (3, 12, 5) + and instruction.opname in ("GET_ITER", "FOR_ITER") + and isinstance(node.parent, ast.For) + and node is node.parent.iter + ): + # node positions have changed in 3.12.5 + # https://github.com/python/cpython/issues/93691 + # `for` calls __iter__ and __next__ during execution, the calling + # expression of these calls was the ast.For node since cpython 3.11 (see test_iter). + # cpython 3.12.5 changed this to the `iter` node of the loop, to make tracebacks easier to read. + # This keeps backward compatibility with older executing versions. + + # there are also cases like: + # + # for a in iter(l): pass + # + # where `iter(l)` would be otherwise the resulting node for the `iter()` call and the __iter__ call of the for implementation. + # keeping the old behaviour makes it possible to distinguish both cases. + + return node.parent + return node + def known_issues(self, node: EnhancedAST, instruction: dis.Instruction) -> None: if instruction.opname in ("COMPARE_OP", "IS_OP", "CONTAINS_OP") and isinstance( node, types_cmp_issue @@ -324,6 +355,35 @@ class PositionNodeFinder(object): ): raise KnownIssue("exception generation maps to condition") + if sys.version_info >= (3, 13): + if instruction.opname in ( + "STORE_FAST_STORE_FAST", + "STORE_FAST_LOAD_FAST", + "LOAD_FAST_LOAD_FAST", + ): + raise KnownIssue(f"can not map {instruction.opname} to two ast nodes") + + if instruction.opname == "LOAD_FAST" and instruction.argval == "__class__": + # example: + # class T: + # def a(): + # super() + # some_node # <- there is a LOAD_FAST for this node because we use super() + + raise KnownIssue( + f"loading of __class__ is accociated with a random node at the end of a class if you use super()" + ) + + if ( + instruction.opname == "COMPARE_OP" + and isinstance(node, ast.UnaryOp) + and isinstance(node.operand,ast.Compare) + and isinstance(node.op, ast.Not) + ): + # work around for + # https://github.com/python/cpython/issues/114671 + self.result = node.operand + @staticmethod def is_except_cleanup(inst: dis.Instruction, node: EnhancedAST) -> bool: if inst.opname not in ( @@ -703,6 +763,52 @@ class PositionNodeFinder(object): if node_match(ast.FormattedValue) and inst_match("FORMAT_VALUE"): return + if sys.version_info >= (3, 13): + + if inst_match("NOP"): + return + + if inst_match("TO_BOOL") and node_match(ast.BoolOp): + return + + if inst_match("CALL_KW") and node_match((ast.Call, ast.ClassDef)): + return + + if inst_match("LOAD_FAST", argval=".type_params"): + return + + if inst_match("LOAD_FAST", argval="__classdict__"): + return + + if inst_match("LOAD_FAST") and node_match( + ( + ast.FunctionDef, + ast.ClassDef, + ast.TypeAlias, + ast.TypeVar, + ast.Lambda, + ast.AsyncFunctionDef, + ) + ): + # These are loads for closure variables. + # It is difficult to check that this is actually closure variable, see: + # https://github.com/alexmojaki/executing/pull/80#discussion_r1716027317 + return + + if ( + inst_match("LOAD_FAST") + and node_match(ast.TypeAlias) + and node.name.id == instruction.argval + ): + return + + if inst_match("STORE_NAME",argval="__static_attributes__"): + # the node is the first node in the body + return + + if inst_match("LOAD_FAST") and isinstance(node.parent,ast.TypeVar): + return + # old verifier @@ -771,11 +877,14 @@ class PositionNodeFinder(object): raise VerifierFailure(title, node, instruction) - def instruction(self, index: int) -> dis.Instruction: - return self.bc_list[index // 2] + def instruction(self, index: int) -> Optional[dis.Instruction]: + return self.bc_dict.get(index,None) def opname(self, index: int) -> str: - return self.instruction(index).opname + i=self.instruction(index) + if i is None: + return "CACHE" + return i.opname extra_node_types=() if sys.version_info >= (3,12): @@ -798,7 +907,10 @@ class PositionNodeFinder(object): *extra_node_types, ), ) -> EnhancedAST: - position = self.instruction(index).positions + instruction = self.instruction(index) + assert instruction is not None + + position = instruction.positions assert position is not None and position.lineno is not None return only( diff --git a/contrib/python/executing/executing/executing.py b/contrib/python/executing/executing/executing.py index 7727c42232..5cf117e18c 100644 --- a/contrib/python/executing/executing/executing.py +++ b/contrib/python/executing/executing/executing.py @@ -273,16 +273,15 @@ class Source(object): node_finder = NodeFinder(frame, stmts, tree, lasti, source) node = node_finder.result decorator = node_finder.decorator + + if node: + new_stmts = {statement_containing_node(node)} + assert_(new_stmts <= stmts) + stmts = new_stmts except Exception: if TESTING: raise - assert stmts is not None - if node: - new_stmts = {statement_containing_node(node)} - assert_(new_stmts <= stmts) - stmts = new_stmts - executing_cache[key] = args = source, node, stmts, decorator return Executing(frame, *args) diff --git a/contrib/python/executing/executing/version.py b/contrib/python/executing/executing/version.py index 9d909dcc3c..b15121b0fe 100644 --- a/contrib/python/executing/executing/version.py +++ b/contrib/python/executing/executing/version.py @@ -1 +1 @@ -__version__ = '2.0.1'
\ No newline at end of file +__version__ = '2.1.0'
\ No newline at end of file diff --git a/contrib/python/executing/ya.make b/contrib/python/executing/ya.make index b676502282..b437b26981 100644 --- a/contrib/python/executing/ya.make +++ b/contrib/python/executing/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(2.0.1) +VERSION(2.1.0) LICENSE(MIT) diff --git a/contrib/python/hypothesis/py3/.dist-info/METADATA b/contrib/python/hypothesis/py3/.dist-info/METADATA index d1e09fb0ef..b78a2eeab3 100644 --- a/contrib/python/hypothesis/py3/.dist-info/METADATA +++ b/contrib/python/hypothesis/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: hypothesis -Version: 6.111.2 +Version: 6.112.0 Summary: A library for property-based testing Home-page: https://hypothesis.works Author: David R. MacIver and Zac Hatfield-Dodds diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/data.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/data.py index 18f3bd9b8a..0f4f1138f3 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/data.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/data.py @@ -111,11 +111,12 @@ class FloatKWargs(TypedDict): class StringKWargs(TypedDict): intervals: IntervalSet min_size: int - max_size: Optional[int] + max_size: int class BytesKWargs(TypedDict): - size: int + min_size: int + max_size: int class BooleanKWargs(TypedDict): @@ -206,7 +207,7 @@ NASTY_FLOATS.extend([-x for x in NASTY_FLOATS]) FLOAT_INIT_LOGIC_CACHE = LRUCache(4096) POOLED_KWARGS_CACHE = LRUCache(4096) -DRAW_STRING_DEFAULT_MAX_SIZE = 10**10 # "arbitrarily large" +COLLECTION_DEFAULT_MAX_SIZE = 10**10 # "arbitrarily large" class Example: @@ -1036,7 +1037,7 @@ class IRNode: return self.value == (minimal_char * self.kwargs["min_size"]) if self.ir_type == "bytes": # smallest size and all-zero value. - return len(self.value) == self.kwargs["size"] and not any(self.value) + return len(self.value) == self.kwargs["min_size"] and not any(self.value) raise NotImplementedError(f"unhandled ir_type {self.ir_type}") @@ -1095,7 +1096,9 @@ def ir_value_permitted(value, ir_type, kwargs): return False return all(ord(c) in kwargs["intervals"] for c in value) elif ir_type == "bytes": - return len(value) == kwargs["size"] + if len(value) < kwargs["min_size"]: + return False + return kwargs["max_size"] is None or len(value) <= kwargs["max_size"] elif ir_type == "boolean": if kwargs["p"] <= 2 ** (-64): return value is False @@ -1314,7 +1317,7 @@ class PrimitiveProvider(abc.ABC): intervals: IntervalSet, *, min_size: int = 0, - max_size: Optional[int] = None, + max_size: int = COLLECTION_DEFAULT_MAX_SIZE, forced: Optional[str] = None, fake_forced: bool = False, ) -> str: @@ -1322,7 +1325,12 @@ class PrimitiveProvider(abc.ABC): @abc.abstractmethod def draw_bytes( - self, size: int, *, forced: Optional[bytes] = None, fake_forced: bool = False + self, + min_size: int = 0, + max_size: int = COLLECTION_DEFAULT_MAX_SIZE, + *, + forced: Optional[bytes] = None, + fake_forced: bool = False, ) -> bytes: raise NotImplementedError @@ -1606,14 +1614,10 @@ class HypothesisProvider(PrimitiveProvider): intervals: IntervalSet, *, min_size: int = 0, - max_size: Optional[int] = None, + max_size: int = COLLECTION_DEFAULT_MAX_SIZE, forced: Optional[str] = None, fake_forced: bool = False, ) -> str: - if max_size is None: - max_size = DRAW_STRING_DEFAULT_MAX_SIZE - - assert forced is None or min_size <= len(forced) <= max_size assert self._cd is not None average_size = min( @@ -1663,17 +1667,40 @@ class HypothesisProvider(PrimitiveProvider): return "".join(chars) def draw_bytes( - self, size: int, *, forced: Optional[bytes] = None, fake_forced: bool = False + self, + min_size: int = 0, + max_size: int = COLLECTION_DEFAULT_MAX_SIZE, + *, + forced: Optional[bytes] = None, + fake_forced: bool = False, ) -> bytes: - forced_i = None - if forced is not None: - forced_i = int_from_bytes(forced) - size = len(forced) - assert self._cd is not None - return self._cd.draw_bits( - 8 * size, forced=forced_i, fake_forced=fake_forced - ).to_bytes(size, "big") + + buf = bytearray() + average_size = min( + max(min_size * 2, min_size + 5), + 0.5 * (min_size + max_size), + ) + elements = many( + self._cd, + min_size=min_size, + max_size=max_size, + average_size=average_size, + forced=None if forced is None else len(forced), + fake_forced=fake_forced, + observe=False, + ) + while elements.more(): + forced_i: Optional[int] = None + if forced is not None: + # implicit conversion from bytes to int by indexing here + forced_i = forced[elements.count - 1] + + buf += self._cd.draw_bits( + 8, forced=forced_i, fake_forced=fake_forced + ).to_bytes(1, "big") + + return bytes(buf) def _draw_float( self, @@ -2216,12 +2243,13 @@ class ConjectureData: intervals: IntervalSet, *, min_size: int = 0, - max_size: Optional[int] = None, + max_size: int = COLLECTION_DEFAULT_MAX_SIZE, forced: Optional[str] = None, fake_forced: bool = False, observe: bool = True, ) -> str: - assert forced is None or min_size <= len(forced) + assert forced is None or min_size <= len(forced) <= max_size + assert min_size >= 0 kwargs: StringKWargs = self._pooled_kwargs( "string", @@ -2255,17 +2283,19 @@ class ConjectureData: def draw_bytes( self, - # TODO move to min_size and max_size here. - size: int, + min_size: int = 0, + max_size: int = COLLECTION_DEFAULT_MAX_SIZE, *, forced: Optional[bytes] = None, fake_forced: bool = False, observe: bool = True, ) -> bytes: - assert forced is None or len(forced) == size - assert size >= 0 + assert forced is None or min_size <= len(forced) <= max_size + assert min_size >= 0 - kwargs: BytesKWargs = self._pooled_kwargs("bytes", {"size": size}) + kwargs: BytesKWargs = self._pooled_kwargs( + "bytes", {"min_size": min_size, "max_size": max_size} + ) if self.ir_tree_nodes is not None and observe: node_value = self._pop_ir_tree_node("bytes", kwargs, forced=forced) diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/datatree.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/datatree.py index 60c1610bc3..87a0c988c8 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/datatree.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/datatree.py @@ -146,9 +146,31 @@ class Conclusion: MAX_CHILDREN_EFFECTIVELY_INFINITE = 100_000 -def compute_max_children(ir_type, kwargs): - from hypothesis.internal.conjecture.data import DRAW_STRING_DEFAULT_MAX_SIZE +def _count_distinct_strings(*, alphabet_size, min_size, max_size): + # We want to estimate if we're going to have more children than + # MAX_CHILDREN_EFFECTIVELY_INFINITE, without computing a potentially + # extremely expensive pow. We'll check if the number of strings in + # the largest string size alone is enough to put us over this limit. + # We'll also employ a trick of estimating against log, which is cheaper + # than computing a pow. + # + # x = max_size + # y = alphabet_size + # n = MAX_CHILDREN_EFFECTIVELY_INFINITE + # + # x**y > n + # <=> log(x**y) > log(n) + # <=> y * log(x) > log(n) + definitely_too_large = max_size * math.log(alphabet_size) > math.log( + MAX_CHILDREN_EFFECTIVELY_INFINITE + ) + if definitely_too_large: + return MAX_CHILDREN_EFFECTIVELY_INFINITE + return sum(alphabet_size**k for k in range(min_size, max_size + 1)) + + +def compute_max_children(ir_type, kwargs): if ir_type == "integer": min_value = kwargs["min_value"] max_value = kwargs["max_value"] @@ -178,50 +200,27 @@ def compute_max_children(ir_type, kwargs): return 1 return 2 elif ir_type == "bytes": - return 2 ** (8 * kwargs["size"]) + return _count_distinct_strings( + alphabet_size=2**8, min_size=kwargs["min_size"], max_size=kwargs["max_size"] + ) elif ir_type == "string": min_size = kwargs["min_size"] max_size = kwargs["max_size"] intervals = kwargs["intervals"] - if max_size is None: - max_size = DRAW_STRING_DEFAULT_MAX_SIZE - if len(intervals) == 0: # Special-case the empty alphabet to avoid an error in math.log(0). # Only possibility is the empty string. return 1 - # We want to estimate if we're going to have more children than - # MAX_CHILDREN_EFFECTIVELY_INFINITE, without computing a potentially - # extremely expensive pow. We'll check if the number of strings in - # the largest string size alone is enough to put us over this limit. - # We'll also employ a trick of estimating against log, which is cheaper - # than computing a pow. - # - # x = max_size - # y = len(intervals) - # n = MAX_CHILDREN_EFFECTIVELY_INFINITE - # - # x**y > n - # <=> log(x**y) > log(n) - # <=> y * log(x) > log(n) - - # avoid math.log(1) == 0 and incorrectly failing the below estimate, - # even when we definitely are too large. - if len(intervals) == 1: - definitely_too_large = max_size > MAX_CHILDREN_EFFECTIVELY_INFINITE - else: - definitely_too_large = max_size * math.log(len(intervals)) > math.log( - MAX_CHILDREN_EFFECTIVELY_INFINITE - ) - - if definitely_too_large: + # avoid math.log(1) == 0 and incorrectly failing our effectively_infinite + # estimate, even when we definitely are too large. + if len(intervals) == 1 and max_size > MAX_CHILDREN_EFFECTIVELY_INFINITE: return MAX_CHILDREN_EFFECTIVELY_INFINITE - # number of strings of length k, for each k in [min_size, max_size]. - return sum(len(intervals) ** k for k in range(min_size, max_size + 1)) - + return _count_distinct_strings( + alphabet_size=len(intervals), min_size=min_size, max_size=max_size + ) elif ir_type == "float": min_value = kwargs["min_value"] max_value = kwargs["max_value"] @@ -306,8 +305,8 @@ def all_children(ir_type, kwargs): else: yield from [False, True] if ir_type == "bytes": - size = kwargs["size"] - yield from (int_to_bytes(i, size) for i in range(2 ** (8 * size))) + for size in range(kwargs["min_size"], kwargs["max_size"] + 1): + yield from (int_to_bytes(i, size) for i in range(2 ** (8 * size))) if ir_type == "string": min_size = kwargs["min_size"] max_size = kwargs["max_size"] diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinker.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinker.py index 3be703a18e..d1084e4cdd 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinker.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinker.py @@ -1075,10 +1075,9 @@ class Shrinker: return False # pragma: no cover if node.ir_type in {"string", "bytes"}: - size_kwarg = "min_size" if node.ir_type == "string" else "size" # if the size *increased*, we would have to guess what to pad with # in order to try fixing up this attempt. Just give up. - if node.kwargs[size_kwarg] <= attempt_kwargs[size_kwarg]: + if node.kwargs["min_size"] <= attempt_kwargs["min_size"]: return False # the size decreased in our attempt. Try again, but replace with # the min_size that we would have gotten, and truncate the value @@ -1089,7 +1088,7 @@ class Shrinker: initial_attempt[node.index].copy( with_kwargs=attempt_kwargs, with_value=initial_attempt[node.index].value[ - : attempt_kwargs[size_kwarg] + : attempt_kwargs["min_size"] ], ) ] diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/bytes.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/bytes.py index 3ba75a2719..7fbc26fd24 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/bytes.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/bytes.py @@ -8,17 +8,16 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -from hypothesis.internal.compat import int_from_bytes, int_to_bytes +from hypothesis.internal.conjecture.shrinking.collection import Collection from hypothesis.internal.conjecture.shrinking.integer import Integer -class Bytes(Integer): +class Bytes(Collection): def __init__(self, initial, predicate, **kwargs): - # shrink by interpreting the bytes as an integer. - # move to Collection.shrink when we support variable-size bytes, - # because b'\x00\x02' could shrink to either b'\x00\x01' or b'\x02'. super().__init__( - int_from_bytes(initial), - lambda n: predicate(int_to_bytes(n, len(initial))), + # implicit conversion from bytes to list of integers here + list(initial), + lambda val: predicate(bytes(val)), + ElementShrinker=Integer, **kwargs, ) diff --git a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/strings.py b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/strings.py index b6e6dd8deb..53d81b7394 100644 --- a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/strings.py +++ b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/strings.py @@ -12,15 +12,16 @@ import copy import re import warnings from functools import lru_cache, partial +from typing import Optional from hypothesis.errors import HypothesisWarning, InvalidArgument from hypothesis.internal import charmap +from hypothesis.internal.conjecture.data import COLLECTION_DEFAULT_MAX_SIZE from hypothesis.internal.filtering import max_len, min_len from hypothesis.internal.intervalsets import IntervalSet from hypothesis.internal.reflection import get_pretty_function_description from hypothesis.strategies._internal.collections import ListStrategy from hypothesis.strategies._internal.lazy import unwrap_strategies -from hypothesis.strategies._internal.numbers import IntegersStrategy from hypothesis.strategies._internal.strategies import ( OneOfStrategy, SampledFromStrategy, @@ -158,7 +159,13 @@ class TextStrategy(ListStrategy): elems = unwrap_strategies(self.element_strategy) if isinstance(elems, OneCharStringStrategy): return data.draw_string( - elems.intervals, min_size=self.min_size, max_size=self.max_size + elems.intervals, + min_size=self.min_size, + max_size=( + COLLECTION_DEFAULT_MAX_SIZE + if self.max_size == float("inf") + else self.max_size + ), ) return "".join(super().do_draw(data)) @@ -224,9 +231,13 @@ def _string_filter_rewrite(self, kind, condition): stacklevel=2, ) - elems = unwrap_strategies(self.element_strategy) if ( - (kind is bytes or isinstance(elems, OneCharStringStrategy)) + ( + kind is bytes + or isinstance( + unwrap_strategies(self.element_strategy), OneCharStringStrategy + ) + ) and isinstance(pattern := getattr(condition, "__self__", None), re.Pattern) and isinstance(pattern.pattern, kind) ): @@ -331,15 +342,15 @@ def _identifier_characters(): return id_start, id_continue -class BytesStrategy(ListStrategy): - def __init__(self, min_size, max_size): - super().__init__(IntegersStrategy(0, 255), min_size=min_size, max_size=max_size) +class BytesStrategy(SearchStrategy): + def __init__(self, min_size: int, max_size: Optional[int]): + self.min_size = min_size + self.max_size = ( + max_size if max_size is not None else COLLECTION_DEFAULT_MAX_SIZE + ) def do_draw(self, data): - # TODO: refactor the underlying provider to support variable-length bytes - if self.min_size == self.max_size: - return bytes(data.draw_bytes(self.min_size)) - return bytes(super().do_draw(data)) + return data.draw_bytes(self.min_size, self.max_size) _nonempty_filters = ( *ListStrategy._nonempty_filters, @@ -353,4 +364,4 @@ class BytesStrategy(ListStrategy): def filter(self, condition): if (new := _string_filter_rewrite(self, bytes, condition)) is not None: return new - return super().filter(condition) + return ListStrategy.filter(self, condition) diff --git a/contrib/python/hypothesis/py3/hypothesis/version.py b/contrib/python/hypothesis/py3/hypothesis/version.py index af1b866e64..2c69dd8977 100644 --- a/contrib/python/hypothesis/py3/hypothesis/version.py +++ b/contrib/python/hypothesis/py3/hypothesis/version.py @@ -8,5 +8,5 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -__version_info__ = (6, 111, 2) +__version_info__ = (6, 112, 0) __version__ = ".".join(map(str, __version_info__)) diff --git a/contrib/python/hypothesis/py3/ya.make b/contrib/python/hypothesis/py3/ya.make index f7183e57f1..ce520eaa5f 100644 --- a/contrib/python/hypothesis/py3/ya.make +++ b/contrib/python/hypothesis/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(6.111.2) +VERSION(6.112.0) LICENSE(MPL-2.0) diff --git a/contrib/python/more-itertools/py3/.dist-info/METADATA b/contrib/python/more-itertools/py3/.dist-info/METADATA index c346b40880..a06c9b0a57 100644 --- a/contrib/python/more-itertools/py3/.dist-info/METADATA +++ b/contrib/python/more-itertools/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: more-itertools -Version: 10.4.0 +Version: 10.5.0 Summary: More routines for operating on iterables, beyond itertools Keywords: itertools,iterator,iteration,filter,peek,peekable,chunk,chunked Author-email: Erik Rose <erikrose@grinchcentral.com> diff --git a/contrib/python/more-itertools/py3/more_itertools/__init__.py b/contrib/python/more-itertools/py3/more_itertools/__init__.py index 2e2fcbbe7b..583fb57457 100644 --- a/contrib/python/more-itertools/py3/more_itertools/__init__.py +++ b/contrib/python/more-itertools/py3/more_itertools/__init__.py @@ -3,4 +3,4 @@ from .more import * # noqa from .recipes import * # noqa -__version__ = '10.4.0' +__version__ = '10.5.0' diff --git a/contrib/python/more-itertools/py3/more_itertools/more.py b/contrib/python/more-itertools/py3/more_itertools/more.py index 3bf2c76b76..64fab26185 100644 --- a/contrib/python/more-itertools/py3/more_itertools/more.py +++ b/contrib/python/more-itertools/py3/more_itertools/more.py @@ -3017,7 +3017,7 @@ def circular_shifts(iterable, steps=1): n = len(buffer) n //= math.gcd(n, steps) - for __ in repeat(None, n): + for _ in repeat(None, n): buffer.rotate(steps) yield tuple(buffer) diff --git a/contrib/python/more-itertools/py3/more_itertools/more.pyi b/contrib/python/more-itertools/py3/more_itertools/more.pyi index f1a155dce7..66e6938e13 100644 --- a/contrib/python/more-itertools/py3/more_itertools/more.pyi +++ b/contrib/python/more-itertools/py3/more_itertools/more.pyi @@ -3,8 +3,8 @@ from __future__ import annotations import sys +import types -from types import TracebackType from typing import ( Any, Callable, @@ -42,7 +42,7 @@ _Raisable = BaseException | Type[BaseException] # The type of isinstance's second argument (from typeshed builtins) if sys.version_info >= (3, 10): - _ClassInfo = type | UnionType | tuple[_ClassInfo, ...] + _ClassInfo = type | types.UnionType | tuple[_ClassInfo, ...] else: _ClassInfo = type | tuple[_ClassInfo, ...] @@ -619,7 +619,7 @@ class callback_iter(Generic[_T], Iterator[_T]): self, exc_type: Type[BaseException] | None, exc_value: BaseException | None, - traceback: TracebackType | None, + traceback: types.TracebackType | None, ) -> bool | None: ... def __iter__(self) -> callback_iter[_T]: ... def __next__(self) -> _T: ... diff --git a/contrib/python/more-itertools/py3/more_itertools/recipes.py b/contrib/python/more-itertools/py3/more_itertools/recipes.py index a21a1f5d88..67f76fa899 100644 --- a/contrib/python/more-itertools/py3/more_itertools/recipes.py +++ b/contrib/python/more-itertools/py3/more_itertools/recipes.py @@ -218,7 +218,12 @@ def all_equal(iterable, key=None): True """ - return len(list(islice(groupby(iterable, key), 2))) <= 1 + iterator = groupby(iterable, key) + for first in iterator: + for second in iterator: + return False + return True + return True def quantify(iterable, pred=bool): diff --git a/contrib/python/more-itertools/py3/tests/test_recipes.py b/contrib/python/more-itertools/py3/tests/test_recipes.py index d3762d49db..684a6fcd0b 100644 --- a/contrib/python/more-itertools/py3/tests/test_recipes.py +++ b/contrib/python/more-itertools/py3/tests/test_recipes.py @@ -2,11 +2,12 @@ from decimal import Decimal from doctest import DocTestSuite from fractions import Fraction from functools import reduce -from itertools import combinations, count, permutations +from itertools import combinations, count, groupby, permutations from operator import mul from math import factorial from sys import version_info from unittest import TestCase, skipIf +from unittest.mock import patch import more_itertools as mi @@ -158,6 +159,22 @@ class AllEqualTests(TestCase): self.assertTrue(mi.all_equal('4٤໔4৪', key=int)) self.assertFalse(mi.all_equal('Abc', key=str.casefold)) + @patch('more_itertools.recipes.groupby', autospec=True) + def test_groupby_calls(self, mock_groupby): + next_count = 0 + + class _groupby(groupby): + def __next__(true_self): + nonlocal next_count + next_count += 1 + return super().__next__() + + mock_groupby.side_effect = _groupby + iterable = iter('aaaaa') + self.assertTrue(mi.all_equal(iterable)) + self.assertEqual(list(iterable), []) + self.assertEqual(next_count, 2) + class QuantifyTests(TestCase): """Tests for ``quantify()``""" diff --git a/contrib/python/more-itertools/py3/ya.make b/contrib/python/more-itertools/py3/ya.make index ee8f86bc14..45df93175b 100644 --- a/contrib/python/more-itertools/py3/ya.make +++ b/contrib/python/more-itertools/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(10.4.0) +VERSION(10.5.0) LICENSE(MIT) diff --git a/contrib/python/types-protobuf/.dist-info/METADATA b/contrib/python/types-protobuf/.dist-info/METADATA index c7742acc49..ce1d526999 100644 --- a/contrib/python/types-protobuf/.dist-info/METADATA +++ b/contrib/python/types-protobuf/.dist-info/METADATA @@ -1,9 +1,9 @@ Metadata-Version: 2.1 Name: types-protobuf -Version: 5.27.0.20240626 +Version: 5.27.0.20240907 Summary: Typing stubs for protobuf Home-page: https://github.com/python/typeshed -License: Apache-2.0 license +License: Apache-2.0 Project-URL: GitHub, https://github.com/python/typeshed Project-URL: Changes, https://github.com/typeshed-internal/stub_uploader/blob/main/data/changelogs/protobuf.md Project-URL: Issue tracker, https://github.com/python/typeshed/issues @@ -38,6 +38,7 @@ If you find that annotations are missing, feel free to contribute and help compl See https://github.com/python/typeshed/blob/main/README.md for more details. -This package was generated from typeshed commit `b13bb947c3f7a000d4d4ec6ad2868726a289b2b4` and was tested -with mypy 1.10.0, pyright 1.1.369, and +This package was generated from typeshed commit +[`e8e9291c76f50c3bcde79e7bb61060f5c24c054e`](https://github.com/python/typeshed/commit/e8e9291c76f50c3bcde79e7bb61060f5c24c054e) and was tested +with mypy 1.11.1, pyright 1.1.379, and pytype 2024.4.11. diff --git a/contrib/python/types-protobuf/google-stubs/protobuf/internal/containers.pyi b/contrib/python/types-protobuf/google-stubs/protobuf/internal/containers.pyi index 30a37353c1..aaa9704392 100644 --- a/contrib/python/types-protobuf/google-stubs/protobuf/internal/containers.pyi +++ b/contrib/python/types-protobuf/google-stubs/protobuf/internal/containers.pyi @@ -33,7 +33,7 @@ class RepeatedScalarFieldContainer(BaseContainer[_ScalarV]): def append(self, value: _ScalarV) -> None: ... def insert(self, key: int, value: _ScalarV) -> None: ... def extend(self, elem_seq: Iterable[_ScalarV] | None) -> None: ... - def MergeFrom(self: _M, other: _M) -> None: ... + def MergeFrom(self: _M, other: _M | Iterable[_ScalarV]) -> None: ... def remove(self, elem: _ScalarV) -> None: ... def pop(self, key: int = -1) -> _ScalarV: ... @overload @@ -49,7 +49,7 @@ class RepeatedCompositeFieldContainer(BaseContainer[_MessageV]): def append(self, value: _MessageV) -> None: ... def insert(self, key: int, value: _MessageV) -> None: ... def extend(self, elem_seq: Iterable[_MessageV]) -> None: ... - def MergeFrom(self: _M, other: _M) -> None: ... + def MergeFrom(self: _M, other: _M | Iterable[_MessageV]) -> None: ... def remove(self, elem: _MessageV) -> None: ... def pop(self, key: int = -1) -> _MessageV: ... def __delitem__(self, key: int | slice) -> None: ... diff --git a/contrib/python/types-protobuf/ya.make b/contrib/python/types-protobuf/ya.make index a4a8846b2c..f2e94d656a 100644 --- a/contrib/python/types-protobuf/ya.make +++ b/contrib/python/types-protobuf/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(5.27.0.20240626) +VERSION(5.27.0.20240907) LICENSE(Apache-2.0) diff --git a/contrib/python/ydb/py3/.dist-info/METADATA b/contrib/python/ydb/py3/.dist-info/METADATA index db2f0036b3..7c7c7cdbaf 100644 --- a/contrib/python/ydb/py3/.dist-info/METADATA +++ b/contrib/python/ydb/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: ydb -Version: 3.17.1 +Version: 3.17.2 Summary: YDB Python SDK Home-page: http://github.com/ydb-platform/ydb-python-sdk Author: Yandex LLC diff --git a/contrib/python/ydb/py3/ya.make b/contrib/python/ydb/py3/ya.make index c1ab6d4472..b3e9929330 100644 --- a/contrib/python/ydb/py3/ya.make +++ b/contrib/python/ydb/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(3.17.1) +VERSION(3.17.2) LICENSE(Apache-2.0) diff --git a/contrib/python/ydb/py3/ydb/table.py b/contrib/python/ydb/py3/ydb/table.py index cfcffb17af..01f5e52b6a 100644 --- a/contrib/python/ydb/py3/ydb/table.py +++ b/contrib/python/ydb/py3/ydb/table.py @@ -290,6 +290,7 @@ class TableIndex(object): self._pb.name = name self.name = name self.index_columns = [] + self.data_columns = [] # output only. self.status = None @@ -307,6 +308,12 @@ class TableIndex(object): self.index_columns.append(column) return self + def with_data_columns(self, *columns): + for column in columns: + self._pb.data_columns.append(column) + self.data_columns.append(column) + return self + def to_pb(self): return self._pb diff --git a/contrib/python/ydb/py3/ydb/ydb_version.py b/contrib/python/ydb/py3/ydb/ydb_version.py index b0ef9f368d..1115cbbcb9 100644 --- a/contrib/python/ydb/py3/ydb/ydb_version.py +++ b/contrib/python/ydb/py3/ydb/ydb_version.py @@ -1 +1 @@ -VERSION = "3.17.1" +VERSION = "3.17.2" |