diff options
Diffstat (limited to 'contrib/python')
178 files changed, 792 insertions, 17166 deletions
diff --git a/contrib/python/aiohappyeyeballs/.dist-info/METADATA b/contrib/python/aiohappyeyeballs/.dist-info/METADATA index c8a61b093c8..440a107f391 100644 --- a/contrib/python/aiohappyeyeballs/.dist-info/METADATA +++ b/contrib/python/aiohappyeyeballs/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.4 Name: aiohappyeyeballs -Version: 2.6.2 +Version: 2.7.1 Summary: Happy Eyeballs for asyncio License: PSF-2.0 License-File: LICENSE @@ -17,6 +17,7 @@ Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 Classifier: License :: OSI Approved :: Python Software Foundation License Project-URL: Bug Tracker, https://github.com/aio-libs/aiohappyeyeballs/issues Project-URL: Changelog, https://github.com/aio-libs/aiohappyeyeballs/blob/main/CHANGELOG.md diff --git a/contrib/python/aiohappyeyeballs/aiohappyeyeballs/__init__.py b/contrib/python/aiohappyeyeballs/aiohappyeyeballs/__init__.py index ad6bfb5a073..d7b322d104e 100644 --- a/contrib/python/aiohappyeyeballs/aiohappyeyeballs/__init__.py +++ b/contrib/python/aiohappyeyeballs/aiohappyeyeballs/__init__.py @@ -1,4 +1,4 @@ -__version__ = "2.6.2" +__version__ = "2.7.1" from .impl import start_connection from .types import AddrInfoType, SocketFactoryType diff --git a/contrib/python/aiohappyeyeballs/aiohappyeyeballs/impl.py b/contrib/python/aiohappyeyeballs/aiohappyeyeballs/impl.py index 4112ab4d1b2..9544ea39887 100644 --- a/contrib/python/aiohappyeyeballs/aiohappyeyeballs/impl.py +++ b/contrib/python/aiohappyeyeballs/aiohappyeyeballs/impl.py @@ -1,11 +1,11 @@ """Base implementation.""" import asyncio -import collections import contextlib import functools import itertools import socket +from collections import defaultdict from collections.abc import Sequence from . import _staggered @@ -209,18 +209,9 @@ async def _connect_sock( raise OSError(f"no matching local address with {family=} found") await loop.sock_connect(sock, address) return sock - except (RuntimeError, OSError) as exc: - my_exceptions.append(exc) - if sock is not None: - if open_sockets is not None: - open_sockets.remove(sock) - try: - sock.close() - except OSError as e: - my_exceptions.append(e) - raise - raise - except: + except BaseException as exc: + if isinstance(exc, (RuntimeError, OSError)): + my_exceptions.append(exc) if sock is not None: if open_sockets is not None: open_sockets.remove(sock) @@ -239,14 +230,9 @@ def _interleave_addrinfos( ) -> list[AddrInfoType]: """Interleave list of addrinfo tuples by family.""" # Group addresses by family - addrinfos_by_family: collections.OrderedDict[int, list[AddrInfoType]] = ( - collections.OrderedDict() - ) + addrinfos_by_family: defaultdict[int, list[AddrInfoType]] = defaultdict(list) for addr in addrinfos: - family = addr[0] - if family not in addrinfos_by_family: - addrinfos_by_family[family] = [] - addrinfos_by_family[family].append(addr) + addrinfos_by_family[addr[0]].append(addr) addrinfos_lists = list(addrinfos_by_family.values()) reordered: list[AddrInfoType] = [] diff --git a/contrib/python/aiohappyeyeballs/aiohappyeyeballs/utils.py b/contrib/python/aiohappyeyeballs/aiohappyeyeballs/utils.py index 5969691ce36..689414b72f1 100644 --- a/contrib/python/aiohappyeyeballs/aiohappyeyeballs/utils.py +++ b/contrib/python/aiohappyeyeballs/aiohappyeyeballs/utils.py @@ -40,19 +40,17 @@ def pop_addr_infos_interleave( The interleave parameter is used to know how many addr_infos for each family should be popped of the top of the list. """ - seen: dict[int, int] = {} if interleave is None: interleave = 1 - to_remove: list[AddrInfoType] = [] + seen: dict[int, int] = {} + kept: list[AddrInfoType] = [] for addr_info in addr_infos: family = addr_info[0] - if family not in seen: - seen[family] = 0 - if seen[family] < interleave: - to_remove.append(addr_info) - seen[family] += 1 - for addr_info in to_remove: - addr_infos.remove(addr_info) + count = seen.get(family, 0) + if count >= interleave: + kept.append(addr_info) + seen[family] = count + 1 + addr_infos[:] = kept def _addr_tuple_to_ip_address( @@ -72,21 +70,13 @@ def remove_addr_infos( The addr value is typically the return value of sock.getpeername(). """ - bad_addrs_infos: list[AddrInfoType] = [] - for addr_info in addr_infos: - if addr_info[-1] == addr: - bad_addrs_infos.append(addr_info) - if bad_addrs_infos: - for bad_addr_info in bad_addrs_infos: - addr_infos.remove(bad_addr_info) - return - # Slow path in case addr is formatted differently - match_addr = _addr_tuple_to_ip_address(addr) - for addr_info in addr_infos: - if match_addr == _addr_tuple_to_ip_address(addr_info[-1]): - bad_addrs_infos.append(addr_info) - if bad_addrs_infos: - for bad_addr_info in bad_addrs_infos: - addr_infos.remove(bad_addr_info) - return - raise ValueError(f"Address {addr} not found in addr_infos") + kept = [ai for ai in addr_infos if ai[-1] != addr] + if len(kept) == len(addr_infos): + # Slow path in case addr is formatted differently + match_addr = _addr_tuple_to_ip_address(addr) + kept = [ + ai for ai in addr_infos if _addr_tuple_to_ip_address(ai[-1]) != match_addr + ] + if len(kept) == len(addr_infos): + raise ValueError(f"Address {addr} not found in addr_infos") + addr_infos[:] = kept diff --git a/contrib/python/aiohappyeyeballs/ya.make b/contrib/python/aiohappyeyeballs/ya.make index 90e2cbfc73d..9054efe4adc 100644 --- a/contrib/python/aiohappyeyeballs/ya.make +++ b/contrib/python/aiohappyeyeballs/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(2.6.2) +VERSION(2.7.1) LICENSE(Python-2.0) diff --git a/contrib/python/argcomplete/py3/.dist-info/METADATA b/contrib/python/argcomplete/py3/.dist-info/METADATA index 33d51a548ba..fa5e1d45d3a 100644 --- a/contrib/python/argcomplete/py3/.dist-info/METADATA +++ b/contrib/python/argcomplete/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.4 Name: argcomplete -Version: 3.6.3 +Version: 3.7.0 Summary: Bash tab completion for argparse Project-URL: Homepage, https://github.com/kislyuk/argcomplete Project-URL: Documentation, https://kislyuk.github.io/argcomplete @@ -22,12 +22,12 @@ Classifier: Operating System :: MacOS :: MacOS X Classifier: Operating System :: POSIX Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy Classifier: Topic :: Software Development diff --git a/contrib/python/argcomplete/py3/.dist-info/entry_points.txt b/contrib/python/argcomplete/py3/.dist-info/entry_points.txt index f13e435fafb..874a9e1481d 100644 --- a/contrib/python/argcomplete/py3/.dist-info/entry_points.txt +++ b/contrib/python/argcomplete/py3/.dist-info/entry_points.txt @@ -1,4 +1,3 @@ [console_scripts] activate-global-python-argcomplete = argcomplete.scripts.activate_global_python_argcomplete:main -python-argcomplete-check-easy-install-script = argcomplete.scripts.python_argcomplete_check_easy_install_script:main register-python-argcomplete = argcomplete.scripts.register_python_argcomplete:main diff --git a/contrib/python/argcomplete/py3/argcomplete/bash_completion.d/_python-argcomplete b/contrib/python/argcomplete/py3/argcomplete/bash_completion.d/_python-argcomplete index 1155f1ce5e8..6d395c66ed9 100644 --- a/contrib/python/argcomplete/py3/argcomplete/bash_completion.d/_python-argcomplete +++ b/contrib/python/argcomplete/py3/argcomplete/bash_completion.d/_python-argcomplete @@ -190,10 +190,7 @@ _python_argcomplete_global() { interpreter=($interpreter) fi - if (__python_argcomplete_scan_head_noerr "$SCRIPT_NAME" easy_install \ - && "${interpreter[@]}" "$(__python_argcomplete_which python-argcomplete-check-easy-install-script)" "$SCRIPT_NAME") >/dev/null 2>&1; then - ARGCOMPLETE=1 - elif ([[ "${interpreter[@]}" == *python* ]] || [[ "${interpreter[@]}" == *pypy* ]])\ + if ([[ "${interpreter[@]}" == *python* ]] || [[ "${interpreter[@]}" == *pypy* ]])\ && __python_argcomplete_run "${interpreter[@]}" -m argcomplete._check_console_script "$SCRIPT_NAME"; then ARGCOMPLETE=1 fi diff --git a/contrib/python/argcomplete/py3/argcomplete/completers.py b/contrib/python/argcomplete/py3/argcomplete/completers.py index 4c01e695189..48715811898 100644 --- a/contrib/python/argcomplete/py3/argcomplete/completers.py +++ b/contrib/python/argcomplete/py3/argcomplete/completers.py @@ -4,6 +4,7 @@ import argparse import os import subprocess +from shlex import quote def _call(*args, **kwargs): @@ -64,20 +65,22 @@ class FilesCompleter(BaseCompleter): # correctly in older versions and calling bind makes them available. For details, see # https://savannah.gnu.org/support/index.php?111125 files = _call( - ["bash", "-c", "bind; compgen -A directory -- '{p}'".format(p=prefix)], stderr=subprocess.DEVNULL + ["bash", "-c", "bind; compgen -A directory -- {p}".format(p=quote(prefix))], + stderr=subprocess.DEVNULL, ) completion += [f + "/" for f in files] for x in self.allowednames: completion += _call( - ["bash", "-c", "bind; compgen -A file -X '!*.{0}' -- '{p}'".format(x, p=prefix)], + ["bash", "-c", "bind; compgen -A file -X '!*.{0}' -- {p}".format(x, p=quote(prefix))], stderr=subprocess.DEVNULL, ) else: completion += _call( - ["bash", "-c", "bind; compgen -A file -- '{p}'".format(p=prefix)], stderr=subprocess.DEVNULL + ["bash", "-c", "bind; compgen -A file -- {p}".format(p=quote(prefix))], stderr=subprocess.DEVNULL ) anticomp = _call( - ["bash", "-c", "bind; compgen -A directory -- '{p}'".format(p=prefix)], stderr=subprocess.DEVNULL + ["bash", "-c", "bind; compgen -A directory -- {p}".format(p=quote(prefix))], + stderr=subprocess.DEVNULL, ) completion = list(set(completion) - set(anticomp)) diff --git a/contrib/python/argcomplete/py3/argcomplete/finders.py b/contrib/python/argcomplete/py3/argcomplete/finders.py index ab8bb1d90a2..2159b7dea2f 100644 --- a/contrib/python/argcomplete/py3/argcomplete/finders.py +++ b/contrib/python/argcomplete/py3/argcomplete/finders.py @@ -527,7 +527,7 @@ class CompletionFinder(object): # (extended to characters other than the colon). if last_wordbreak_pos is not None: completions = [c[last_wordbreak_pos + 1 :] for c in completions] - special_chars += "();<>|&!`$* \t\n\"'" + special_chars += "();<>|&!`$*?[]{} \t\n\"'" elif cword_prequote == '"': special_chars += '"`$!' diff --git a/contrib/python/argcomplete/py3/argcomplete/packages/_argparse.py b/contrib/python/argcomplete/py3/argcomplete/packages/_argparse.py index f6ecb1f4172..574d9d8207a 100644 --- a/contrib/python/argcomplete/py3/argcomplete/packages/_argparse.py +++ b/contrib/python/argcomplete/py3/argcomplete/packages/_argparse.py @@ -19,7 +19,13 @@ from argparse import ( _SubParsersAction, ) from gettext import gettext -from typing import Dict, List, Set, Tuple +from typing import Dict, List, Optional, Set, Tuple, Union, cast + +_OptionTuple = Union[ + Tuple[Optional[Action], str, Optional[str]], + Tuple[Optional[Action], str, Optional[str], Optional[str]], +] +_OptionTupleEntry = Union[_OptionTuple, List[_OptionTuple]] _num_consumed_args: Dict[Action, int] = {} @@ -97,7 +103,7 @@ class IntrospectiveArgumentParser(ArgumentParser): # find all option indices, and determine the arg_string_pattern # which has an 'O' if there is an option at an index, # an 'A' if there is an argument, or a '-' if there is a '--' - option_string_indices = {} + option_string_indices: Dict[int, _OptionTupleEntry] = {} arg_string_pattern_parts = [] arg_strings_iter = iter(arg_strings) for i, arg_string in enumerate(arg_strings_iter): @@ -114,7 +120,7 @@ class IntrospectiveArgumentParser(ArgumentParser): if option_tuple is None: pattern = 'A' else: - option_string_indices[i] = option_tuple + option_string_indices[i] = cast(_OptionTupleEntry, option_tuple) pattern = 'O' arg_string_pattern_parts.append(pattern) @@ -161,9 +167,11 @@ class IntrospectiveArgumentParser(ArgumentParser): # function to convert arg_strings into an optional action def consume_optional(start_index): # get the optional identified at this index - option_tuple = option_string_indices[start_index] - if isinstance(option_tuple, list): # Python 3.12.7+ - option_tuple = option_tuple[0] + raw_option_tuple = option_string_indices[start_index] + if isinstance(raw_option_tuple, list): # Python 3.12.7+ + option_tuple = raw_option_tuple[0] + else: + option_tuple = raw_option_tuple if len(option_tuple) == 3: action, option_string, explicit_arg = option_tuple else: # Python 3.11.9+, 3.12.3+, 3.13+ @@ -241,8 +249,8 @@ class IntrospectiveArgumentParser(ArgumentParser): # add the Optional to the list and return the index at which # the Optional's string args stopped assert action_tuples - for action, args, option_string in action_tuples: - take_action(action, args, option_string) + for optional_action, args, option_string in action_tuples: + take_action(optional_action, args, option_string) return stop # the list of Positionals left to be parsed; this is modified diff --git a/contrib/python/argcomplete/py3/argcomplete/scripts/python_argcomplete_check_easy_install_script.py b/contrib/python/argcomplete/py3/argcomplete/scripts/python_argcomplete_check_easy_install_script.py deleted file mode 100644 index 0eb744c9e96..00000000000 --- a/contrib/python/argcomplete/py3/argcomplete/scripts/python_argcomplete_check_easy_install_script.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2012-2023, Andrey Kislyuk and argcomplete contributors. -# Licensed under the Apache License. See https://github.com/kislyuk/argcomplete for more info. - -""" -This script is part of the Python argcomplete package (https://github.com/kislyuk/argcomplete). -It is used to check if an EASY-INSTALL-SCRIPT wrapper redirects to a script that contains the string -"PYTHON_ARGCOMPLETE_OK". If you have enabled global completion in argcomplete, the completion hook will run it every -time you press <TAB> in your shell. - -Usage: - python-argcomplete-check-easy-install-script <input executable file> -""" - -import sys - -# PEP 366 -__package__ = "argcomplete.scripts" - - -def main(): - if len(sys.argv) != 2: - sys.exit(__doc__) - - sys.tracebacklimit = 0 - - with open(sys.argv[1]) as fh: - line1, head = fh.read(1024).split("\n", 1)[:2] - if line1.startswith("#") and ("py" in line1 or "Py" in line1): - import re - - lines = head.split("\n", 12) - for line in lines: - if line.startswith("# EASY-INSTALL-SCRIPT"): - import pkg_resources # type: ignore - - re_match = re.match("# EASY-INSTALL-SCRIPT: '(.+)','(.+)'", line) - assert re_match is not None - dist, script = re_match.groups() - if "PYTHON_ARGCOMPLETE_OK" in pkg_resources.get_distribution(dist).get_metadata( - "scripts/" + script - ): - return 0 - elif line.startswith("# EASY-INSTALL-ENTRY-SCRIPT"): - re_match = re.match("# EASY-INSTALL-ENTRY-SCRIPT: '(.+)','(.+)','(.+)'", line) - assert re_match is not None - dist, group, name = re_match.groups() - import pkgutil - - import pkg_resources # type: ignore - - entry_point_info = pkg_resources.get_distribution(dist).get_entry_info(group, name) - assert entry_point_info is not None - module_name = entry_point_info.module_name - with open(pkgutil.get_loader(module_name).get_filename()) as mod_fh: # type: ignore - if "PYTHON_ARGCOMPLETE_OK" in mod_fh.read(1024): - return 0 - elif line.startswith("# EASY-INSTALL-DEV-SCRIPT"): - for line2 in lines: - if line2.startswith("__file__"): - re_match = re.match("__file__ = '(.+)'", line2) - assert re_match is not None - filename = re_match.group(1) - with open(filename) as mod_fh: - if "PYTHON_ARGCOMPLETE_OK" in mod_fh.read(1024): - return 0 - elif line.startswith("# PBR Generated"): - re_match = re.search("from (.*) import", head) - assert re_match is not None - module = re_match.groups()[0] - import pkgutil - - import pkg_resources # type: ignore - - with open(pkgutil.get_loader(module).get_filename()) as mod_fh: # type: ignore - if "PYTHON_ARGCOMPLETE_OK" in mod_fh.read(1024): - return 0 - - return 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/contrib/python/argcomplete/py3/ya.make b/contrib/python/argcomplete/py3/ya.make index 1ffd0082bb2..397e1ff3f2e 100644 --- a/contrib/python/argcomplete/py3/ya.make +++ b/contrib/python/argcomplete/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(3.6.3) +VERSION(3.7.0) LICENSE(Apache-2.0) @@ -23,7 +23,6 @@ PY_SRCS( argcomplete/packages/_shlex.py argcomplete/scripts/__init__.py argcomplete/scripts/activate_global_python_argcomplete.py - argcomplete/scripts/python_argcomplete_check_easy_install_script.py argcomplete/scripts/register_python_argcomplete.py argcomplete/shell_integration.py ) diff --git a/contrib/python/chardet/py2/tests/ya.make b/contrib/python/chardet/py2/tests/ya.make index 3795bfa7fbd..b5e332be138 100644 --- a/contrib/python/chardet/py2/tests/ya.make +++ b/contrib/python/chardet/py2/tests/ya.make @@ -1,5 +1,7 @@ PY2TEST() +SUBSCRIBER(g:python-contrib ) + SRCDIR(contrib/python/chardet/py2) TEST_SRCS( diff --git a/contrib/python/chardet/py3/tests/ya.make b/contrib/python/chardet/py3/tests/ya.make index c1f16c7df7f..16b148c70e9 100644 --- a/contrib/python/chardet/py3/tests/ya.make +++ b/contrib/python/chardet/py3/tests/ya.make @@ -1,5 +1,7 @@ PY3TEST() +SUBSCRIBER(g:python-contrib ) + SRCDIR(contrib/python/chardet/py3) TEST_SRCS( diff --git a/contrib/python/chardet/py3/ya.make b/contrib/python/chardet/py3/ya.make index b14449360dd..14f394da7c6 100644 --- a/contrib/python/chardet/py3/ya.make +++ b/contrib/python/chardet/py3/ya.make @@ -2,6 +2,8 @@ PY3_LIBRARY() +SUBSCRIBER(g:python-contrib) + VERSION(5.2.0) LICENSE(LGPL-2.1-or-later) diff --git a/contrib/python/clickhouse-connect/.dist-info/METADATA b/contrib/python/clickhouse-connect/.dist-info/METADATA index cf511761a8d..656e95416df 100644 --- a/contrib/python/clickhouse-connect/.dist-info/METADATA +++ b/contrib/python/clickhouse-connect/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.4 Name: clickhouse-connect -Version: 1.3.0 +Version: 1.4.1 Summary: ClickHouse Database Core Driver for Python, Pandas, and Superset Home-page: https://github.com/ClickHouse/clickhouse-connect Author: ClickHouse Inc. diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/_version.py b/contrib/python/clickhouse-connect/clickhouse_connect/_version.py index d28b3ddc3b1..6abaa204304 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/_version.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/_version.py @@ -1 +1 @@ -version = "1.3.0" +version = "1.4.1" diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/alembic/adapter.py b/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/alembic/adapter.py index d444b69e73a..0319a329855 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/alembic/adapter.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/alembic/adapter.py @@ -5,6 +5,7 @@ from alembic.operations import Operations, ops from alembic.runtime.migration import MigrationContext from alembic.util import DispatchPriority, PriorityDispatchResult +from clickhouse_connect.cc_sqlalchemy.alembic.impl import ClickHouseImpl from clickhouse_connect.cc_sqlalchemy.datatypes.base import ChSqlaType from clickhouse_connect.cc_sqlalchemy.sql.ddlcompiler import ClickHouseDDLHelper @@ -57,6 +58,12 @@ def clickhouse_writer(context: MigrationContext, revision, directives): _add_common_imports(directive) +def _is_clickhouse_autogen(autogen_context: AutogenContext) -> bool: + """True only when the active migration context targets the ClickHouse dialect.""" + migration_context = getattr(autogen_context, "migration_context", None) + return isinstance(getattr(migration_context, "impl", None), ClickHouseImpl) + + def render_clickhouse_column(column, autogen_context: AutogenContext) -> str: rendered = render._user_defined_render("column", column, autogen_context) if rendered is not False: @@ -98,8 +105,19 @@ def render_clickhouse_column(column, autogen_context: AutogenContext) -> str: ) +# Alembic renderers have no dialect qualifier, so replace=True overrides rendering +# process-wide. Capture each built-in renderer before replacing it and delegate to it for +# non-ClickHouse dialects so autogenerate stays correct for other databases (#832). Held on +# the module so importlib.reload does not re-capture one of our own renderers and recurse. +_DEFAULT_RENDERERS = globals().get("_DEFAULT_RENDERERS") or { + op: render.renderers.dispatch(op) for op in (ops.CreateTableOp, ops.AddColumnOp, ops.DropTableOp) +} + + @render.renderers.dispatch_for(ops.CreateTableOp, replace=True) def render_create_table(autogen_context: AutogenContext, op: ops.CreateTableOp) -> str: + if not _is_clickhouse_autogen(autogen_context): + return _DEFAULT_RENDERERS[ops.CreateTableOp](autogen_context, op) table = op.to_table() args = [column for column in [render_clickhouse_column(column, autogen_context) for column in table.columns] if column] + sorted( @@ -138,6 +156,8 @@ def render_create_table(autogen_context: AutogenContext, op: ops.CreateTableOp) @render.renderers.dispatch_for(ops.AddColumnOp, replace=True) def render_add_column(autogen_context: AutogenContext, op: ops.AddColumnOp) -> str: + if not _is_clickhouse_autogen(autogen_context): + return _DEFAULT_RENDERERS[ops.AddColumnOp](autogen_context, op) schema, table_name, column, if_not_exists = op.schema, op.table_name, op.column, op.if_not_exists prefix = render._alembic_autogenerate_prefix(autogen_context) rendered_column = render_clickhouse_column(column, autogen_context) @@ -155,6 +175,8 @@ def render_add_column(autogen_context: AutogenContext, op: ops.AddColumnOp) -> s @render.renderers.dispatch_for(ops.DropTableOp, replace=True) def render_drop_table(autogen_context: AutogenContext, op: ops.DropTableOp) -> str: + if not _is_clickhouse_autogen(autogen_context): + return _DEFAULT_RENDERERS[ops.DropTableOp](autogen_context, op) prefix = render._alembic_autogenerate_prefix(autogen_context) rendered = f"{prefix}drop_table({render._ident(op.table_name)!r}" arguments = [] diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/alembic/impl.py b/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/alembic/impl.py index 1f942979c07..32e2b985cd4 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/alembic/impl.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/alembic/impl.py @@ -65,7 +65,7 @@ class ClickHouseImpl(DefaultImpl): if self.connection is None: return try: - self.connection.connection.driver_connection.client._add_integration_tag("alembic") + self.connection.connection.driver_connection.client._add_integration_tag("alembic") # type: ignore[union-attr] except Exception: pass @@ -100,7 +100,7 @@ class ClickHouseImpl(DefaultImpl): construct, execution_options=execution_options, multiparams=multiparams, - params=params or {}, + params=params or {}, # type: ignore[arg-type] ) def add_column( diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/alembic/utils.py b/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/alembic/utils.py index 35ff8fb7b94..14ac01c27fc 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/alembic/utils.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/alembic/utils.py @@ -75,9 +75,9 @@ def prevent_empty_migrations(writer_fn: Callable) -> Callable: if not directives: return config = context.config - if getattr(config.cmd_opts, "autogenerate", False): + if config is not None and getattr(config.cmd_opts, "autogenerate", False): script = directives[0] - if script.upgrade_ops.is_empty(): + if script.upgrade_ops is not None and script.upgrade_ops.is_empty(): directives.clear() return writer_fn(context, revision, directives) diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/datatypes/base.py b/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/datatypes/base.py index a4c1d4a94f2..175c203668e 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/datatypes/base.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/datatypes/base.py @@ -15,11 +15,10 @@ class ChSqlaType: subclasses will inherit from TypeEngine. """ - ch_type: ClickHouseType = None + ch_type: ClickHouseType | None = None generic_type: None - _ch_type_cls = None - _instance = None - _instance_cache: dict[TypeDef, "ChSqlaType"] = None + _ch_type_cls: type[ClickHouseType] | None = None + _instance_cache: dict[TypeDef, "ChSqlaType"] | None = None def __init_subclass__(cls): """ @@ -44,7 +43,7 @@ class ChSqlaType: :param type_def: -- TypeDef tuple that defines arguments for this instance :return: Shared instance of a configured ChSqlaType """ - return cls._instance_cache.setdefault(type_def, cls(type_def=type_def)) + return cls._instance_cache.setdefault(type_def, cls(type_def=type_def)) # type: ignore[union-attr] def __init__(self, type_def: TypeDef = EMPTY_TYPE_DEF): """ @@ -55,7 +54,7 @@ class ChSqlaType: parse_name function """ self.type_def = type_def - self.ch_type = self._ch_type_cls.build(type_def) + self.ch_type = self._ch_type_cls.build(type_def) # type: ignore[union-attr] @property def name(self): @@ -124,7 +123,7 @@ class CaseInsensitiveDict(dict): sqla_type_map: dict[str, type[ChSqlaType]] = CaseInsensitiveDict() -schema_types = [] +schema_types: list[str] = [] def sqla_type_from_name(name: str) -> ChSqlaType: diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/datatypes/sqltypes.py b/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/datatypes/sqltypes.py index 5356150545b..ad7ee5ef173 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/datatypes/sqltypes.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/datatypes/sqltypes.py @@ -2,6 +2,7 @@ import ipaddress import uuid from collections.abc import Sequence from enum import Enum as PyEnum +from typing import Any, cast from sqlalchemy.exc import ArgumentError from sqlalchemy.types import ( @@ -10,6 +11,7 @@ from sqlalchemy.types import ( Integer, Interval, Numeric, + TypeEngine, UserDefinedType, ) from sqlalchemy.types import ( @@ -33,67 +35,67 @@ from clickhouse_connect.driver import tzutil from clickhouse_connect.driver.common import decimal_prec -class Int8(ChSqlaType, Integer): +class Int8(ChSqlaType, Integer): # type: ignore[misc] pass -class UInt8(ChSqlaType, Integer): +class UInt8(ChSqlaType, Integer): # type: ignore[misc] pass -class Int16(ChSqlaType, Integer): +class Int16(ChSqlaType, Integer): # type: ignore[misc] pass -class UInt16(ChSqlaType, Integer): +class UInt16(ChSqlaType, Integer): # type: ignore[misc] pass -class Int32(ChSqlaType, Integer): +class Int32(ChSqlaType, Integer): # type: ignore[misc] pass -class UInt32(ChSqlaType, Integer): +class UInt32(ChSqlaType, Integer): # type: ignore[misc] pass -class Int64(ChSqlaType, Integer): +class Int64(ChSqlaType, Integer): # type: ignore[misc] pass -class UInt64(ChSqlaType, Integer): +class UInt64(ChSqlaType, Integer): # type: ignore[misc] pass -class Int128(ChSqlaType, Integer): +class Int128(ChSqlaType, Integer): # type: ignore[misc] pass -class UInt128(ChSqlaType, Integer): +class UInt128(ChSqlaType, Integer): # type: ignore[misc] pass -class Int256(ChSqlaType, Integer): +class Int256(ChSqlaType, Integer): # type: ignore[misc] pass -class UInt256(ChSqlaType, Integer): +class UInt256(ChSqlaType, Integer): # type: ignore[misc] pass -class Float32(ChSqlaType, Float): +class Float32(ChSqlaType, Float): # type: ignore[misc] def __init__(self, type_def: TypeDef = EMPTY_TYPE_DEF): ChSqlaType.__init__(self, type_def) Float.__init__(self) -class Float64(ChSqlaType, Float): +class Float64(ChSqlaType, Float): # type: ignore[misc] def __init__(self, type_def: TypeDef = EMPTY_TYPE_DEF): ChSqlaType.__init__(self, type_def) Float.__init__(self) -class Bool(ChSqlaType, SqlaBoolean): +class Bool(ChSqlaType, SqlaBoolean): # type: ignore[misc] def __init__(self, type_def: TypeDef = EMPTY_TYPE_DEF, **kwargs): ChSqlaType.__init__(self, type_def) SqlaBoolean.__init__(self, **kwargs) @@ -103,10 +105,10 @@ class Boolean(Bool): pass -class Decimal(ChSqlaType, Numeric): +class Decimal(ChSqlaType, Numeric): # type: ignore[misc] dec_size = 0 - def __init__(self, precision: int = 0, scale: int = 0, type_def: TypeDef = None): + def __init__(self, precision: int = 0, scale: int = 0, type_def: TypeDef | None = None): """ Construct either with precision and scale (for DDL), or a TypeDef with those values (by name) :param precision: Number of digits the Decimal @@ -143,16 +145,16 @@ class Decimal256(Decimal): dec_size = 256 -class Enum(ChSqlaType, UserDefinedType): +class Enum(ChSqlaType, UserDefinedType): # type: ignore[misc] _size = 16 python_type = str def __init__( self, - enum: type[PyEnum] = None, - keys: Sequence[str] = None, - values: Sequence[int] = None, - type_def: TypeDef = None, + enum: type[PyEnum] | None = None, + keys: Sequence[str] | None = None, + values: Sequence[int] | None = None, + type_def: TypeDef | None = None, ): """ Construct a ClickHouse enum either from a Python Enum or parallel lists of keys and value. Note that @@ -166,6 +168,8 @@ class Enum(ChSqlaType, UserDefinedType): if enum: keys = [e.name for e in enum] values = [e.value for e in enum] + if keys is None or values is None: + raise ArgumentError("Enum requires either a Python enum or both 'keys' and 'values'") self._validate(keys, values) if self.__class__.__name__ == "Enum": if max(values) <= 127 and min(values) >= -128: @@ -176,7 +180,7 @@ class Enum(ChSqlaType, UserDefinedType): super().__init__(type_def) @classmethod - def _validate(cls, keys: Sequence, values: Sequence): + def _validate(cls, keys: Sequence[str], values: Sequence[int]): bad_key = next((x for x in keys if not isinstance(x, str)), None) if bad_key: raise ArgumentError(f"ClickHouse enum key {bad_key} is not a string") @@ -199,63 +203,63 @@ class Enum16(Enum): _ch_type_cls = ChEnum16 -class String(ChSqlaType, UserDefinedType): +class String(ChSqlaType, UserDefinedType): # type: ignore[misc] python_type = str -class FixedString(ChSqlaType, SqlaString): - def __init__(self, size: int = -1, type_def: TypeDef = None): +class FixedString(ChSqlaType, SqlaString): # type: ignore[misc] + def __init__(self, size: int = -1, type_def: TypeDef | None = None): if not type_def: type_def = TypeDef(values=(size,)) ChSqlaType.__init__(self, type_def) SqlaString.__init__(self, size) -class IPv4(ChSqlaType, UserDefinedType): +class IPv4(ChSqlaType, UserDefinedType): # type: ignore[misc] python_type = ipaddress.IPv4Address -class IPv6(ChSqlaType, UserDefinedType): +class IPv6(ChSqlaType, UserDefinedType): # type: ignore[misc] python_type = ipaddress.IPv6Address -class UUID(ChSqlaType, UserDefinedType): +class UUID(ChSqlaType, UserDefinedType): # type: ignore[misc] python_type = uuid.UUID -class Nothing(ChSqlaType, UserDefinedType): +class Nothing(ChSqlaType, UserDefinedType): # type: ignore[misc] python_type = type(None) -class Point(ChSqlaType, UserDefinedType): +class Point(ChSqlaType, UserDefinedType): # type: ignore[misc] python_type = tuple -class Ring(ChSqlaType, UserDefinedType): +class Ring(ChSqlaType, UserDefinedType): # type: ignore[misc] python_type = list -class Polygon(ChSqlaType, UserDefinedType): +class Polygon(ChSqlaType, UserDefinedType): # type: ignore[misc] python_type = list -class MultiPolygon(ChSqlaType, UserDefinedType): +class MultiPolygon(ChSqlaType, UserDefinedType): # type: ignore[misc] python_type = list -class LineString(ChSqlaType, UserDefinedType): +class LineString(ChSqlaType, UserDefinedType): # type: ignore[misc] python_type = list -class MultiLineString(ChSqlaType, UserDefinedType): +class MultiLineString(ChSqlaType, UserDefinedType): # type: ignore[misc] python_type = list -class Date(ChSqlaType, SqlaDate): +class Date(ChSqlaType, SqlaDate): # type: ignore[misc] pass -class Date32(ChSqlaType, SqlaDate): +class Date32(ChSqlaType, SqlaDate): # type: ignore[misc] pass @@ -283,8 +287,8 @@ def _resolve_tz_alias(tz, timezone): return tz -class DateTime(ChSqlaType, SqlaDateTime): - def __init__(self, tz: str = None, type_def: TypeDef = None, timezone=_TIMEZONE_SENTINEL): +class DateTime(ChSqlaType, SqlaDateTime): # type: ignore[misc] + def __init__(self, tz: str | None = None, type_def: TypeDef | None = None, timezone=_TIMEZONE_SENTINEL): """tz / timezone: IANA zone string (resolved via zoneinfo; install `tzdata` on Windows).""" tz = _resolve_tz_alias(tz, timezone) if not type_def: @@ -297,8 +301,8 @@ class DateTime(ChSqlaType, SqlaDateTime): SqlaDateTime.__init__(self) -class DateTime64(ChSqlaType, SqlaDateTime): - def __init__(self, precision: int = None, tz: str = None, type_def: TypeDef = None, timezone=_TIMEZONE_SENTINEL): +class DateTime64(ChSqlaType, SqlaDateTime): # type: ignore[misc] + def __init__(self, precision: int | None = None, tz: str | None = None, type_def: TypeDef | None = None, timezone=_TIMEZONE_SENTINEL): """precision: 3/6/9 for ms/us/ns. tz / timezone: IANA zone string.""" tz = _resolve_tz_alias(tz, timezone) if not type_def: @@ -314,7 +318,7 @@ class DateTime64(ChSqlaType, SqlaDateTime): SqlaDateTime.__init__(self) -class Time(ChSqlaType, Interval): +class Time(ChSqlaType, Interval): # type: ignore[misc] """ Represents the ClickHouse Time type, which corresponds to a timedelta. @@ -336,7 +340,7 @@ class Time(ChSqlaType, Interval): return None -class Time64(ChSqlaType, Interval): +class Time64(ChSqlaType, Interval): # type: ignore[misc] """ Represents the ClickHouse Time64 type with configurable precision. @@ -345,7 +349,7 @@ class Time64(ChSqlaType, Interval): If no precision is defined it default to 3. """ - def __init__(self, precision: int = None, type_def: TypeDef = None): + def __init__(self, precision: int | None = None, type_def: TypeDef | None = None): """ Time64 constructor with precision if not constructed with TypeDef. :param precision: 3 (ms), 6 (us), or 9 (ns) for sub-second precision. @@ -375,51 +379,29 @@ class Time64(ChSqlaType, Interval): return None -class Nullable: - """ - Class "wrapper" to use in DDL construction. It is never actually initialized but instead creates the "wrapped" - type with a Nullable wrapper - """ +def Nullable(element: ChSqlaType | type[ChSqlaType]) -> ChSqlaType: # noqa: N802 + """Wrap a ChSqlaType instance or class with a Nullable modifier for DDL construction.""" + if callable(element): + return element(type_def=NULLABLE_TYPE_DEF) + orig = element.type_def + wrappers = orig if "Nullable" in orig.wrappers else orig.wrappers + ("Nullable",) + return element.__class__(type_def=TypeDef(wrappers, orig.keys, orig.values)) - def __new__(cls, element: ChSqlaType | type[ChSqlaType]): - """ - Actually returns an instance of the enclosed type with a Nullable wrapper. If element is an instance, - constructs a new instance with a copied TypeDef plus the Nullable wrapper. If element is just a type, - constructs a new element of that type with only the Nullable wrapper. - :param element: ChSqlaType instance or class to wrap - """ - if callable(element): - return element(type_def=NULLABLE_TYPE_DEF) - orig = element.type_def - wrappers = orig if "Nullable" in orig.wrappers else orig.wrappers + ("Nullable",) - return element.__class__(type_def=TypeDef(wrappers, orig.keys, orig.values)) +def LowCardinality(element: ChSqlaType | type[ChSqlaType]) -> ChSqlaType: # noqa: N802 + """Wrap a ChSqlaType instance or class with a LowCardinality modifier for DDL construction.""" + if callable(element): + return element(type_def=LC_TYPE_DEF) + orig = element.type_def + wrappers = orig if "LowCardinality" in orig.wrappers else ("LowCardinality",) + orig.wrappers + return element.__class__(type_def=TypeDef(wrappers, orig.keys, orig.values)) -class LowCardinality: - """ - Class "wrapper" to use in DDL construction. It is never actually instantiated but instead creates the "wrapped" - type with a LowCardinality wrapper - """ - def __new__(cls, element: ChSqlaType | type[ChSqlaType]): - """ - Actually returns an instance of the enclosed type with a LowCardinality wrapper. If element is an instance, - constructs a new instance with a copied TypeDef plus the LowCardinality wrapper. If element is just a type, - constructs a new element of that type with only the LowCardinality wrapper. - :param element: ChSqlaType instance or class to wrap - """ - if callable(element): - return element(type_def=LC_TYPE_DEF) - orig = element.type_def - wrappers = orig if "LowCardinality" in orig.wrappers else ("LowCardinality",) + orig.wrappers - return element.__class__(type_def=TypeDef(wrappers, orig.keys, orig.values)) - - -class Array(ChSqlaType, ARRAY): +class Array(ChSqlaType, ARRAY): # type: ignore[misc] python_type = list dimensions = 1 - def __init__(self, element: ChSqlaType | type[ChSqlaType] = None, type_def: TypeDef = None): + def __init__(self, element: ChSqlaType | type[ChSqlaType] | None = None, type_def: TypeDef | None = None): """ Array constructor that can take a wrapped Array type if not constructed from a TypeDef :param element: ChSqlaType instance or class to wrap @@ -428,23 +410,25 @@ class Array(ChSqlaType, ARRAY): if not type_def: if callable(element): element = element() + if element is None: + raise ArgumentError("Array requires an element type or type_def") type_def = TypeDef(values=(element.name,)) ChSqlaType.__init__(self, type_def) # Set item_type directly; calling ARRAY.__init__ would reject nested Array(Array(T)), # which CH supports natively (CH expresses dimensions via nesting, not a dim count). # as_tuple has no class-level default, so set it here to satisfy ARRAY result processing. - self.item_type = sqla_type_from_name(type_def.values[0]) + self.item_type = cast("TypeEngine[Any]", sqla_type_from_name(type_def.values[0])) self.as_tuple = False -class Map(ChSqlaType, UserDefinedType): +class Map(ChSqlaType, UserDefinedType): # type: ignore[misc] python_type = dict def __init__( self, - key_type: ChSqlaType | type[ChSqlaType] = None, - value_type: ChSqlaType | type[ChSqlaType] = None, - type_def: TypeDef = None, + key_type: ChSqlaType | type[ChSqlaType] | None = None, + value_type: ChSqlaType | type[ChSqlaType] | None = None, + type_def: TypeDef | None = None, ): """ Map constructor that can take a wrapped key/values types if not constructed from a TypeDef @@ -457,18 +441,20 @@ class Map(ChSqlaType, UserDefinedType): key_type = key_type() if callable(value_type): value_type = value_type() + if key_type is None or value_type is None: + raise ArgumentError("Map requires key_type and value_type, or type_def") type_def = TypeDef(values=(key_type.name, value_type.name)) super().__init__(type_def) -class Tuple(ChSqlaType, UserDefinedType): +class Tuple(ChSqlaType, UserDefinedType): # type: ignore[misc] python_type = tuple def __init__( self, *args, - elements: Sequence[ChSqlaType | type[ChSqlaType]] = None, - type_def: TypeDef = None, + elements: Sequence[ChSqlaType | type[ChSqlaType]] | None = None, + type_def: TypeDef | None = None, ): """Tuple(UInt32, UUID) variadic form or Tuple(elements=[UInt32, UUID]) list form, not both.""" if type_def is None and not args and elements is None: @@ -481,7 +467,7 @@ class Tuple(ChSqlaType, UserDefinedType): raise ArgumentError("Cannot specify both positional elements and the 'elements' kwarg") if args: elements = args - values = [et() if callable(et) else et for et in elements] + values = [et() if callable(et) else et for et in elements] # type: ignore[union-attr] type_def = TypeDef(values=tuple(v.name for v in values)) super().__init__(type_def) @@ -493,7 +479,7 @@ class Tuple(ChSqlaType, UserDefinedType): return inst -class JSON(ChSqlaType, UserDefinedType): +class JSON(ChSqlaType, UserDefinedType): # type: ignore[misc] """ Note this isn't currently supported for insert/select, only table definitions """ @@ -501,7 +487,7 @@ class JSON(ChSqlaType, UserDefinedType): python_type = dict -class Nested(ChSqlaType, UserDefinedType): +class Nested(ChSqlaType, UserDefinedType): # type: ignore[misc] """ Note this isn't currently supported for insert/select, only table definitions """ @@ -509,14 +495,14 @@ class Nested(ChSqlaType, UserDefinedType): python_type = list -class SimpleAggregateFunction(ChSqlaType, UserDefinedType): +class SimpleAggregateFunction(ChSqlaType, UserDefinedType): # type: ignore[misc] python_type = str def __init__( self, - name: str = None, - element: ChSqlaType | type[ChSqlaType] = None, - type_def: TypeDef = None, + name: str | None = None, + element: ChSqlaType | type[ChSqlaType] | None = None, + type_def: TypeDef | None = None, ): """ Constructor that can take the SimpleAggregateFunction name and wrapped type if not constructed from a TypeDef @@ -527,23 +513,20 @@ class SimpleAggregateFunction(ChSqlaType, UserDefinedType): if not type_def: if callable(element): element = element() - type_def = TypeDef( - values=( - name, - element.name, - ) - ) + if element is None: + raise ArgumentError("SimpleAggregateFunction requires an element type or type_def") + type_def = TypeDef(values=(name, element.name)) super().__init__(type_def) -class AggregateFunction(ChSqlaType, UserDefinedType): +class AggregateFunction(ChSqlaType, UserDefinedType): # type: ignore[misc] """ Note this isn't currently supported for insert/select, only table definitions """ python_type = str - def __init__(self, *params, type_def: TypeDef = None): + def __init__(self, *params, type_def: TypeDef | None = None): """ Simply wraps the parameters for AggregateFunction for DDL, unless the TypeDef is specified. Callables or actual types are converted to their names. @@ -551,7 +534,7 @@ class AggregateFunction(ChSqlaType, UserDefinedType): :param type_def: TypeDef from parse_name function """ if not type_def: - values = () + values: tuple[Any, ...] = () for x in params: if callable(x): x = x() @@ -562,10 +545,10 @@ class AggregateFunction(ChSqlaType, UserDefinedType): super().__init__(type_def) -class QBit(ChSqlaType, UserDefinedType): +class QBit(ChSqlaType, UserDefinedType): # type: ignore[misc] python_type = list - def __init__(self, element_type: str = None, dimension: int = None, type_def: TypeDef = None): + def __init__(self, element_type: str | None = None, dimension: int | None = None, type_def: TypeDef | None = None): """ QBit constructor for bit-transposed vector types :param element_type: Element type (BFloat16, Float32, or Float64) diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/ddl/custom.py b/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/ddl/custom.py index b29e8e6b26a..2e7187ed6d6 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/ddl/custom.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/ddl/custom.py @@ -12,8 +12,8 @@ class CreateDatabase(DDL): def __init__( self, name: str, - engine: str = None, - zoo_path: str = None, + engine: str | None = None, + zoo_path: str | None = None, shard_name: str = "{shard}", replica_name: str = "{replica}", exists_ok: bool = False, diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/ddl/tableengine.py b/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/ddl/tableengine.py index 85759cfc6df..358c86aa0a5 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/ddl/tableengine.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/ddl/tableengine.py @@ -4,9 +4,8 @@ from typing import Any from sqlalchemy import Column from sqlalchemy.exc import ArgumentError, SQLAlchemyError -from sqlalchemy.sql.base import SchemaEventTarget from sqlalchemy.sql.elements import TextClause -from sqlalchemy.sql.visitors import Visitable +from sqlalchemy.sql.schema import SchemaItem from clickhouse_connect.cc_sqlalchemy.sql.sqlparse import split_top_level, walk_sql from clickhouse_connect.driver.binding import format_str, quote_identifier @@ -36,7 +35,7 @@ def _render_setting_value(value: Any) -> str: return format_str(str(value)) -def tuple_expr(expr_name, value: EngineParam): +def tuple_expr(expr_name: str, value: EngineParam) -> str: """ Create a table parameter with a tuple or list correctly formatted :param expr_name: parameter @@ -48,7 +47,7 @@ def tuple_expr(expr_name, value: EngineParam): v = f"{expr_name.strip()}" if isinstance(value, (tuple, list)): return f" {v} ({','.join(_render_engine_expr(item) for item in value)})" - return f"{v} {_render_engine_expr(value)}" + return f"{v} {_render_engine_expr(value)}" # type: ignore[arg-type] def repr_engine_value(value: Any) -> str: @@ -68,22 +67,22 @@ def repr_engine_value(value: Any) -> str: return repr(value) -class TableEngine(SchemaEventTarget, Visitable): +class TableEngine(SchemaItem): """ SqlAlchemy Schema element to support ClickHouse table engines. At the moment provides no real functionality other than the CREATE TABLE argument string """ - arg_names = () - quoted_args = set() - optional_args = set() - eng_params = () + arg_names: Sequence[str] = () + quoted_args: set[str] = set() + optional_args: set[str] = set() + eng_params: Sequence[str] = () def __init_subclass__(cls, **kwargs): engine_map[cls.__name__] = cls def __init__(self, kwargs): - Visitable.__init__(self) + super().__init__() self.name = self.__class__.__name__ te_name = f"{self.name} Table Engine" self._orig_kwargs = kwargs.copy() @@ -167,21 +166,21 @@ class Set(TableEngine): class Dictionary(TableEngine): arg_names = ["dictionary"] - def __init__(self, dictionary: str = None): + def __init__(self, dictionary: str | None = None): super().__init__(locals()) class Merge(TableEngine): arg_names = ["db_name, tables_regexp"] - def __init__(self, db_name: str = None, tables_regexp: str = None): + def __init__(self, db_name: str | None = None, tables_regexp: str | None = None): super().__init__(locals()) class File(TableEngine): arg_names = ["fmt"] - def __init__(self, fmt: str = None): + def __init__(self, fmt: str | None = None): super().__init__(locals()) @@ -189,7 +188,14 @@ class Distributed(TableEngine): arg_names = ["cluster", "database", "table", "sharding_key", "policy_name"] optional_args = {"sharding_key", "policy_name"} - def __init__(self, cluster: str = None, database: str = None, table=None, sharding_key: str = None, policy_name: str = None): + def __init__( + self, + cluster: str | None = None, + database: str | None = None, + table=None, + sharding_key: str | None = None, + policy_name: str | None = None, + ): super().__init__(locals()) @@ -229,9 +235,9 @@ class ReplacingMergeTree(TableEngine): def __init__( self, - ver: str = None, - version: str = None, - is_deleted: str = None, + ver: str | None = None, + version: str | None = None, + is_deleted: str | None = None, order_by: EngineParam = None, primary_key: EngineParam = None, partition_by: EngineParam = None, @@ -260,7 +266,7 @@ class CollapsingMergeTree(TableEngine): def __init__( self, - sign: str = None, + sign: str | None = None, order_by: EngineParam = None, primary_key: EngineParam = None, partition_by: EngineParam = None, @@ -279,8 +285,8 @@ class VersionedCollapsingMergeTree(TableEngine): def __init__( self, - sign: str = None, - version: str = None, + sign: str | None = None, + version: str | None = None, order_by: EngineParam = None, primary_key: EngineParam = None, partition_by: EngineParam = None, @@ -300,8 +306,8 @@ class GraphiteMergeTree(TableEngine): def __init__( self, - config_section: str = None, - version: str = None, + config_section: str | None = None, + version: str | None = None, order_by: EngineParam = None, primary_key: EngineParam = None, partition_by: EngineParam = None, @@ -326,8 +332,8 @@ class ReplicatedMergeTree(TableEngine): primary_key: EngineParam = None, partition_by: EngineParam = None, sample_by: EngineParam = None, - zk_path: str = None, - replica: str = None, + zk_path: str | None = None, + replica: str | None = None, ttl: EngineExpr | None = None, settings: dict[str, Any] | None = None, ): @@ -352,13 +358,13 @@ class ReplicatedReplacingMergeTree(TableEngine): def __init__( self, - ver: str = None, + ver: str | None = None, order_by: EngineParam = None, primary_key: EngineParam = None, partition_by: EngineParam = None, sample_by: EngineParam = None, - zk_path: str = None, - replica: str = None, + zk_path: str | None = None, + replica: str | None = None, ttl: EngineExpr | None = None, settings: dict[str, Any] | None = None, ): @@ -375,13 +381,13 @@ class ReplicatedCollapsingMergeTree(TableEngine): def __init__( self, - sign: str = None, + sign: str | None = None, order_by: EngineParam = None, primary_key: EngineParam = None, partition_by: EngineParam = None, sample_by: EngineParam = None, - zk_path: str = None, - replica: str = None, + zk_path: str | None = None, + replica: str | None = None, ttl: EngineExpr | None = None, settings: dict[str, Any] | None = None, ): @@ -398,14 +404,14 @@ class ReplicatedVersionedCollapsingMergeTree(TableEngine): def __init__( self, - sign: str = None, - version: str = None, + sign: str | None = None, + version: str | None = None, order_by: EngineParam = None, primary_key: EngineParam = None, partition_by: EngineParam = None, sample_by: EngineParam = None, - zk_path: str = None, - replica: str = None, + zk_path: str | None = None, + replica: str | None = None, ttl: EngineExpr | None = None, settings: dict[str, Any] | None = None, ): @@ -422,13 +428,13 @@ class ReplicatedGraphiteMergeTree(TableEngine): def __init__( self, - config_section: str = None, + config_section: str | None = None, order_by: EngineParam = None, primary_key: EngineParam = None, partition_by: EngineParam = None, sample_by: EngineParam = None, - zk_path: str = None, - replica: str = None, + zk_path: str | None = None, + replica: str | None = None, ttl: EngineExpr | None = None, settings: dict[str, Any] | None = None, ): diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/dialect.py b/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/dialect.py index 368e80422ba..bdfd529af8d 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/dialect.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/dialect.py @@ -31,7 +31,7 @@ class ClickHouseDialect(DefaultDialect): postfetch_lastrowid = False ddl_compiler = ChDDLCompiler statement_compiler = ChStatementCompiler - preparer = ChIdentifierPreparer + preparer = ChIdentifierPreparer # type: ignore[assignment] description_encoding = None max_identifier_length = 127 ischema_names = ischema_names diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/sql/__init__.py b/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/sql/__init__.py index ec2cee4c942..3352936603f 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/sql/__init__.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/sql/__init__.py @@ -1,3 +1,5 @@ +from typing import Any + from sqlalchemy import Table, and_ from sqlalchemy.sql.selectable import FromClause, Select @@ -16,7 +18,7 @@ def full_table(table_name: str, schema: str | None = None) -> str: return f"{quote_identifier(schema)}.{quote_identifier(table_name)}" -def format_table(table: Table): +def format_table(table: Table) -> str: return full_table(table.name, table.schema) @@ -45,21 +47,21 @@ def _resolve_target(select_stmt: Select, table: FromClause | None, method_name: def _target_cache_key(target: FromClause) -> str: if hasattr(target, "fullname"): - return target.fullname - return target.name + return target.fullname # type: ignore[attr-defined] + return target.name # type: ignore[attr-defined] def final(select_stmt: Select, table: FromClause | None = None) -> Select: """Apply the ClickHouse FINAL modifier. For ReplacingMergeTree-family engines.""" target = _resolve_target(select_stmt, table, "final") - ch_final = getattr(select_stmt, "_ch_final", set()) + ch_final: set[FromClause] = getattr(select_stmt, "_ch_final", set()) if target in ch_final: return select_stmt hint_key = _target_cache_key(target) new_stmt = select_stmt.with_statement_hint(f"FINAL:{hint_key}", dialect_name=_CH_MODIFIER_DIALECT) - new_stmt._ch_final = ch_final | {target} + new_stmt._ch_final = ch_final | {target} # type: ignore[attr-defined] return new_stmt @@ -75,7 +77,7 @@ def sample(select_stmt: Select, sample_value: str | int | float, table: FromClau new_stmt = select_stmt.with_statement_hint(f"SAMPLE:{hint_key}:{sample_value}", dialect_name=_CH_MODIFIER_DIALECT) ch_sample = dict(getattr(select_stmt, "_ch_sample", {})) ch_sample[target] = sample_value - new_stmt._ch_sample = ch_sample + new_stmt._ch_sample = ch_sample # type: ignore[attr-defined] return new_stmt @@ -83,7 +85,7 @@ def _select_sample(self: Select, sample_value: str | int | float, table: FromCla return sample(self, sample_value=sample_value, table=table) -def _apply_array_join(select_stmt: Select, cols, alias, is_left: bool) -> Select: +def _apply_array_join(select_stmt: Select, cols: Any, alias: Any, is_left: bool) -> Select: if not isinstance(select_stmt, Select): raise TypeError("array_join() expects a SQLAlchemy Select instance") @@ -125,7 +127,7 @@ def _select_left_array_join(self: Select, *cols, alias=None) -> Select: return _apply_array_join(self, cols, alias, is_left=True) -def prewhere(select_stmt, whereclause): +def prewhere(select_stmt: Select, whereclause: Any) -> Select: """Apply ClickHouse PREWHERE. Multiple calls compose with AND.""" if not isinstance(select_stmt, Select): raise TypeError("prewhere() expects a SQLAlchemy Select instance") @@ -136,11 +138,11 @@ def prewhere(select_stmt, whereclause): # Hint key is str(combined) (structural, with bind placeholders) rather # than id() so equivalent statements share a compiled-statement cache entry. new_stmt = select_stmt.with_statement_hint(f"PREWHERE:{str(combined)}", dialect_name=_CH_MODIFIER_DIALECT) - new_stmt._ch_prewhere = PreWhereClause(combined) + new_stmt._ch_prewhere = PreWhereClause(combined) # type: ignore[attr-defined] return new_stmt -def limit_by(select_stmt, by_clauses, limit, offset=None): +def limit_by(select_stmt: Select, by_clauses: Any, limit: int, offset: int | None = None) -> Select: """Apply ClickHouse LIMIT BY (top-N per group). Renders `LIMIT [offset,] limit BY by_clauses`.""" if not isinstance(select_stmt, Select): raise TypeError("limit_by() expects a SQLAlchemy Select instance") @@ -151,21 +153,21 @@ def limit_by(select_stmt, by_clauses, limit, offset=None): by_key = ",".join(str(c) for c in by_tuple) new_stmt = select_stmt.with_statement_hint(f"LIMIT_BY:{limit}:{offset}:{by_key}", dialect_name=_CH_MODIFIER_DIALECT) - new_stmt._ch_limit_by = LimitByClause(by_tuple, limit, offset) + new_stmt._ch_limit_by = LimitByClause(by_tuple, limit, offset) # type: ignore[attr-defined] return new_stmt -def _select_prewhere(self, whereclause): +def _select_prewhere(self: Select, whereclause: Any) -> Select: return prewhere(self, whereclause) -def _select_limit_by(self, by_clauses, limit, offset=None): +def _select_limit_by(self: Select, by_clauses: Any, limit: int, offset: int | None = None) -> Select: return limit_by(self, by_clauses, limit, offset) -Select.sample = _select_sample -Select.final = _select_final -Select.array_join = _select_array_join -Select.left_array_join = _select_left_array_join -Select.prewhere = _select_prewhere -Select.limit_by = _select_limit_by +Select.sample = _select_sample # type: ignore[attr-defined] +Select.final = _select_final # type: ignore[attr-defined] +Select.array_join = _select_array_join # type: ignore[attr-defined] +Select.left_array_join = _select_left_array_join # type: ignore[attr-defined] +Select.prewhere = _select_prewhere # type: ignore[attr-defined] +Select.limit_by = _select_limit_by # type: ignore[attr-defined] diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/sql/preparer.py b/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/sql/preparer.py index 4521eb97dfa..cda0f5a2ed8 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/sql/preparer.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/cc_sqlalchemy/sql/preparer.py @@ -4,7 +4,7 @@ from clickhouse_connect.driver.binding import quote_identifier class ChIdentifierPreparer(IdentifierPreparer): - quote_identifier = staticmethod(quote_identifier) + quote_identifier = staticmethod(quote_identifier) # type: ignore[assignment] def __init__(self, dialect, **kwargs): super().__init__(dialect, **kwargs) diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/base.py b/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/base.py index 220a2dda579..ab5e17740c4 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/base.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/base.py @@ -16,8 +16,8 @@ from clickhouse_connect.driver.query import QueryContext from clickhouse_connect.driver.types import ByteSource logger = logging.getLogger(__name__) -ch_read_formats = {} -ch_write_formats = {} +ch_read_formats: dict[type, str] = {} +ch_write_formats: dict[type, str] = {} class TypeDef(NamedTuple): @@ -45,10 +45,10 @@ class ClickHouseType(ABC): # noqa: B024 np_type = "O" # Default to Numpy Object type nano_divisor = 0 # Only relevant for date like objects byte_size = 0 - valid_formats = "native" + valid_formats: str | tuple[str, ...] = "native" - python_type = None - base_type = None + python_type: type | None = None + base_type: str | None = None @property def _null_time_unit(self): @@ -110,7 +110,7 @@ class ClickHouseType(ABC): # noqa: B024 def insert_name(self): return self.name - def data_size(self, sample: Sequence) -> int: + def data_size(self, sample: Collection) -> int: if self.low_card: values = set(sample) d_size = self._data_size(values) + 2 @@ -126,7 +126,7 @@ class ClickHouseType(ABC): # noqa: B024 total = 0 for x in sample: total += len(str(x)) - return total / len(sample) + 1 + return total // len(sample) + 1 def write_column_prefix(self, dest: bytearray): """ @@ -264,9 +264,9 @@ class ClickHouseType(ABC): # noqa: B024 def _write_column_low_card(self, column: Sequence, dest: bytearray, ctx: InsertContext): if len(column) == 0: return - keys = [] - index = [] - rev_map = {} + keys: list[int] = [] + index: list[Any] = [] + rev_map: dict[Any, int] = {} rmg = rev_map.get if self.nullable: index.append(None) @@ -319,15 +319,16 @@ class ArrayType(ClickHouseType, ABC, registered=False): """ _signed = True - _array_type = None - _struct_type = None + _array_type: str | None = None + _struct_type: str | None = None valid_formats = "string", "native" - python_type = int + python_type: type = int def __init_subclass__(cls, registered: bool = True): super().__init_subclass__(registered) if cls._array_type in ("i", "I") and int_size == 2: - cls._array_type = "L" if cls._array_type.isupper() else "l" + array_type_char = cls._array_type + cls._array_type = "L" if array_type_char.isupper() else "l" if isinstance(cls._array_type, str) and cls._array_type: cls._struct_type = "<" + cls._array_type cls.byte_size = array.array(cls._array_type).itemsize @@ -335,14 +336,17 @@ class ArrayType(ClickHouseType, ABC, registered=False): def _read_column_binary(self, source: ByteSource, num_rows: int, ctx: QueryContext, _read_state: Any): if ctx.use_numpy: return driver_ctypes.numpy_conv.read_numpy_array(source, self.np_type, num_rows) + assert self._array_type is not None return source.read_array(self._array_type, num_rows) def _read_nullable_column(self, source: ByteSource, num_rows: int, ctx: QueryContext, _read_state: Any) -> Sequence: + assert self._array_type is not None return data_conv.read_nullable_array(source, self._array_type, num_rows, self._active_null(ctx)) def _build_lc_column(self, index: Sequence, keys: array.array, ctx: QueryContext): if ctx.use_numpy: - return options.np.fromiter((index[key] for key in keys), dtype=index.dtype, count=len(index)) + # index is a numpy array when ctx.use_numpy is True + return options.np.fromiter((index[key] for key in keys), dtype=index.dtype, count=len(index)) # type: ignore[attr-defined] return super()._build_lc_column(index, keys, ctx) def _finalize_column(self, column: Sequence, ctx: QueryContext) -> Sequence: @@ -357,6 +361,7 @@ class ArrayType(ClickHouseType, ABC, registered=False): def _write_column_binary(self, column: Sequence | MutableSequence, dest: bytearray, ctx: InsertContext): if len(column) and self.nullable: column = [0 if x is None else x for x in column] + assert self._array_type is not None write_array(self._array_type, column, dest, ctx.column_name) def _active_null(self, ctx: QueryContext): @@ -377,7 +382,7 @@ class UnsupportedType(ClickHouseType, ABC, registered=False): super().__init__(type_def) self._name_suffix = type_def.arg_str - def _read_column_binary(self, source: Sequence, num_rows: int, ctx: QueryContext, read_state: Any): + def _read_column_binary(self, _source: ByteSource, _num_rows: int, _ctx: QueryContext, _read_state: Any): raise NotSupportedError(f"{self.name} deserialization not supported") def _write_column_binary(self, column: Sequence | MutableSequence, dest: bytearray, ctx: InsertContext): diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/container.py b/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/container.py index 88e87540695..951e618d623 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/container.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/container.py @@ -33,7 +33,7 @@ class Array(ClickHouseType): def read_column_prefix(self, source: ByteSource, ctx: QueryContext): return self.element_type.read_column_prefix(source, ctx) - def _data_size(self, sample: Sequence) -> int: + def _data_size(self, sample: Collection[Any]) -> int: if len(sample) == 0: return 8 total = 0 @@ -139,7 +139,7 @@ class Tuple(ClickHouseType): column = e_type.read_column_data(source, num_rows, ctx, read_state[ix]) columns.append(column) if e_names and self.read_format(ctx) != "tuple": - dicts = [{} for _ in range(num_rows)] + dicts: list[dict[str, Any]] = [{} for _ in range(num_rows)] for ix, x in enumerate(dicts): for y, key in enumerate(e_names): x[key] = columns[y][ix] @@ -163,7 +163,7 @@ class Tuple(ClickHouseType): def convert_dict_insert(self, column: Sequence) -> Sequence: names = self.element_names - col = [[] for _ in names] + col: list[list[Any]] = [[] for _ in names] for x in column: for ix, name in enumerate(names): col[ix].append(x.get(name)) diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/dynamic.py b/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/dynamic.py index 150fa1ced1f..4732ff1b957 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/dynamic.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/dynamic.py @@ -156,7 +156,7 @@ class Variant(ClickHouseType): v_count = len(self.element_types) if v_count == 0: return 1 - sub_samples = [[] for _ in range(v_count)] + sub_samples: list[list[Any]] = [[] for _ in range(v_count)] for v in sample: if v is None: continue @@ -197,7 +197,7 @@ def read_variant_column( # Now we have to walk through each of the discriminators again to assign the correct value from # the sub-column to the final result column sub_indexes = [0] * v_count - col = [] + col: list[Any] = [] app_col = col.append for disc in discriminators: if disc == 255: @@ -230,7 +230,9 @@ def read_dynamic_prefix(_, source: ByteSource, ctx: QueryContext) -> DynamicStat class Dynamic(ClickHouseType): python_type = object - read_column_prefix = read_dynamic_prefix + + def read_column_prefix(self, source: ByteSource, ctx: QueryContext) -> DynamicState: + return read_dynamic_prefix(self, source, ctx) @property def insert_name(self): @@ -272,7 +274,7 @@ def write_json(ch_type: ClickHouseType, column: Sequence, dest: bytearray, ctx: first = first_value(column, ch_type.nullable) write_col = column - encoding = ctx.encoding or ch_type.encoding + encoding: str | None = ctx.encoding or ch_type.encoding if not isinstance(first, str) and ch_type.write_format(ctx) != "string": to_json = any_to_json if ch_type.nullable: @@ -576,7 +578,7 @@ class JSON(ClickHouseType): shared_columns = SHARED_DATA_TYPE.read_column_data(source, num_rows, ctx, read_state.shared_state) # noqa: F821 (undefined-name) col = [] for row_num in range(num_rows): - top = {} + top: dict[str, Any] = {} for ix, field in enumerate(self.typed_paths): _nest_value(top, field, typed_columns[ix][row_num]) for ix, field in enumerate(read_state.dynamic_paths): diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/format.py b/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/format.py index fa09e2c9fdd..9bb02e72b7d 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/format.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/format.py @@ -63,7 +63,7 @@ def _convert_arguments(*args, **kwargs) -> dict[str, str]: return fmt_map -def _matching_types(pattern: str, fmt: str = None) -> Sequence[type[ClickHouseType]]: +def _matching_types(pattern: str, fmt: str | None = None) -> Sequence[type[ClickHouseType]]: re_pattern = re.compile(pattern.replace("*", ".*"), re.IGNORECASE) matches = [ch_type for type_name, ch_type in type_map.items() if re_pattern.match(type_name)] if not matches: diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/network.py b/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/network.py index e0e751cf390..69f200dd7b9 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/network.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/network.py @@ -67,22 +67,24 @@ class IPv6(ClickHouseType): """Read IPv6 addresses in native format, always returning IPv6Address objects.""" fast_ip_v6 = IPv6Address.__new__ with_scope_id = "_scope_id" in IPv6Address.__slots__ - new_col = [] + new_col: list[IPv6Address] = [] app = new_col.append ifb = int.from_bytes for _ in range(num_rows): int_value = ifb(source.read_bytes(16), "big") ipv6 = fast_ip_v6(IPv6Address) - ipv6._ip = int_value + # Bypass IPv6Address.__init__ for performance; _ip and _scope_id are + # the internal representation used by CPython's ipaddress module. + ipv6._ip = int_value # type: ignore[attr-defined] if with_scope_id: - ipv6._scope_id = None + ipv6._scope_id = None # type: ignore[attr-defined] app(ipv6) return new_col @staticmethod def _read_binary_str(source: ByteSource, num_rows: int) -> list[str]: """Read IPv6 addresses in string format, always returning IPv6Address strings.""" - new_col = [] + new_col: list[str] = [] app = new_col.append tov6 = socket.inet_ntop af6 = socket.AF_INET6 diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/numeric.py b/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/numeric.py index 37b0d83ed4c..90d2abea7ab 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/numeric.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/numeric.py @@ -16,6 +16,8 @@ from clickhouse_connect.driver.types import ByteSource class IntBase(ArrayType, registered=False): + _array_type: str + def _write_column_binary(self, column: Sequence | MutableSequence, dest: bytearray, ctx: InsertContext): if len(column) == 0: return @@ -109,7 +111,7 @@ class BigInt(ClickHouseType, registered=False): def _read_column_binary(self, source: ByteSource, num_rows: int, ctx: QueryContext, _read_state: Any): signed = self._signed sz = self.byte_size - column = [] + column: list[Any] = [] app = column.append ifb = int.from_bytes if self.read_format(ctx) == "string": @@ -407,7 +409,7 @@ class BigDecimal(Decimal, registered=False): def _read_column_binary(self, source: ByteSource, num_rows: int, _ctx: QueryContext, _read_state: Any): dec = decimal.Decimal scale = self.scale - column = [] + column: list[Any] = [] app = column.append sz = self.byte_size ifb = int.from_bytes diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/registry.py b/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/registry.py index 3f5e45866c8..cb035782c85 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/registry.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/registry.py @@ -1,4 +1,5 @@ import logging +from typing import Any from clickhouse_connect.datatypes.base import ClickHouseType, TypeDef, type_map from clickhouse_connect.driver.exceptions import InternalError @@ -18,7 +19,8 @@ def parse_name(name: str) -> tuple[str, str, TypeDef]: """ base = name wrappers = [] - keys = tuple() + keys: tuple[Any, ...] = () + values: tuple[Any, ...] = () if base.startswith("LowCardinality"): wrappers.append("LowCardinality") base = base[15:-1] diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/special.py b/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/special.py index 7b030fb3662..f5f5df9f45d 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/special.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/special.py @@ -1,4 +1,4 @@ -from collections.abc import MutableSequence, Sequence +from collections.abc import Collection, MutableSequence, Sequence from typing import Any from uuid import UUID as PYUUID @@ -30,7 +30,7 @@ class UUID(ClickHouseType): @staticmethod def _read_binary_str(source: ByteSource, num_rows: int): v = source.read_array("Q", num_rows * 2) - column = [] + column: list[str] = [] app = column.append for i in range(num_rows): ix = i << 1 @@ -94,7 +94,7 @@ class SimpleAggregateFunction(ClickHouseType): self.python_type = self.element_type.python_type self.nano_divisor = self.element_type.nano_divisor - def _data_size(self, sample: Sequence) -> int: + def _data_size(self, sample: Collection[Any]) -> int: return self.element_type.data_size(sample) def read_column_prefix(self, source: ByteSource, ctx: QueryContext): diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/temporal.py b/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/temporal.py index 0f8ed6ad43f..c2105643afb 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/temporal.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/temporal.py @@ -4,9 +4,9 @@ import array import re import zoneinfo from abc import abstractmethod -from collections.abc import MutableSequence, Sequence +from collections.abc import Callable, MutableSequence, Sequence from datetime import date, datetime, time, timedelta, tzinfo -from typing import TYPE_CHECKING, Any, NamedTuple +from typing import TYPE_CHECKING, Any, NamedTuple, cast if TYPE_CHECKING: import numpy @@ -50,6 +50,7 @@ class Date(ClickHouseType): if self.nullable: column = [x if x else 0 for x in column] else: + esd: date if isinstance(first, datetime): esd = epoch_start_datetime else: @@ -112,6 +113,7 @@ class Date32(Date): class DateTimeBase(ClickHouseType, registered=False): __slots__ = ("tzinfo",) + tzinfo: tzinfo | None valid_formats = "native", "int" python_type = datetime @@ -143,12 +145,13 @@ class DateTimeBase(ClickHouseType, registered=False): if isinstance(column, list): column = options.pd.DatetimeIndex(column) - if column.tz is None: - result = column.astype(self.pandas_dtype) + dti = cast(Any, column) + if dti.tz is None: + result = dti.astype(self.pandas_dtype) return options.pd.array(result) if self.nullable else result - naive_ns = column.tz_convert("UTC").tz_localize(None).astype(self.pandas_dtype) - tz_aware_result = naive_ns.tz_localize("UTC").tz_convert(column.tz) + naive_ns = dti.tz_convert("UTC").tz_localize(None).astype(self.pandas_dtype) + tz_aware_result = naive_ns.tz_localize("UTC").tz_convert(dti.tz) return options.pd.array(tz_aware_result) if self.nullable else tz_aware_result if self.nullable: @@ -329,7 +332,6 @@ class TimeBase(ClickHouseType, registered=False): _array_type: str byte_size: int - np_type: str valid_formats = ("native", "string", "int", "time") python_type = timedelta @@ -403,7 +405,7 @@ class TimeBase(ClickHouseType, registered=False): return [0] * len(column) return [] - converter_map = { + converter_map: dict[type, Callable[..., int]] = { timedelta: self._timedelta_to_ticks, time: self._time_to_ticks, float: self._numerical_to_ticks, @@ -509,7 +511,7 @@ class TimeBase(ClickHouseType, registered=False): raise NotImplementedError @abstractmethod - def _ticks_to_np_timedelta(self, ticks: int) -> timedelta: + def _ticks_to_np_timedelta(self, ticks: int) -> timedelta | numpy.timedelta64: """Convert integer ticks into an np.timedelta.""" raise NotImplementedError @@ -638,7 +640,7 @@ class Time64(TimeBase): return self._SECONDS_PER_DAY * self.precision - 1 @property - def np_type(self) -> str: + def np_type(self): return f"timedelta64{self.unit}" @property diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/vector.py b/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/vector.py index 0d666941f74..e0dff1b56c7 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/vector.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/vector.py @@ -4,7 +4,7 @@ import logging from collections.abc import Sequence from math import ceil, nan from struct import pack, unpack -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast if TYPE_CHECKING: import numpy @@ -89,7 +89,7 @@ class QBit(ClickHouseType): tuple_data = self._tuple_type.read_column_data(source, num_rows, ctx, read_state) vectors = [self._untranspose_row(t) for t in tuple_data] if self.nullable: - return data_conv.build_nullable_column(vectors, null_map, self._active_null(ctx)) + return data_conv.build_nullable_column(vectors, cast(bytes, null_map), self._active_null(ctx)) return vectors def write_column_prefix(self, dest: bytearray): diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/dbapi/__init__.py b/contrib/python/clickhouse-connect/clickhouse_connect/dbapi/__init__.py index 84a9a8c3fcc..99a298d4f0d 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/dbapi/__init__.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/dbapi/__init__.py @@ -1,3 +1,5 @@ +from typing import Any + from clickhouse_connect.dbapi.connection import Connection apilevel = "2.0" # PEP 249 DB API level @@ -12,11 +14,11 @@ class Error(Exception): def connect( host: str | None = None, database: str | None = None, - username: str | None = "", - password: str | None = "", + username: str = "", + password: str = "", port: int | None = None, - **kwargs, -): + **kwargs: Any, +) -> Connection: secure = kwargs.pop("secure", False) return Connection( host=host, diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/dbapi/connection.py b/contrib/python/clickhouse-connect/clickhouse_connect/dbapi/connection.py index 3dba3c54aef..7b0cafaf121 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/dbapi/connection.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/dbapi/connection.py @@ -1,3 +1,5 @@ +from typing import Any + from clickhouse_connect.dbapi.cursor import Cursor from clickhouse_connect.driver import create_client from clickhouse_connect.driver.query import QueryResult @@ -10,15 +12,15 @@ class Connection: def __init__( self, - dsn: str = None, + dsn: str | None = None, username: str = "", password: str = "", - host: str = None, - database: str = None, - interface: str = None, - port: int = 0, + host: str | None = None, + database: str | None = None, + interface: str | None = None, + port: int | None = None, secure: bool | str = False, - **kwargs, + **kwargs: Any, ): self.client = create_client( host=host, @@ -35,20 +37,20 @@ class Connection: self.client._add_integration_tag("sqlalchemy") self.timezone = self.client.server_tz - def close(self): + def close(self) -> None: self.client.close() - def commit(self): + def commit(self) -> None: pass - def rollback(self): + def rollback(self) -> None: pass - def command(self, cmd: str): + def command(self, cmd: str) -> Any: return self.client.command(cmd) def raw_query(self, query: str) -> QueryResult: return self.client.query(query) - def cursor(self): + def cursor(self) -> Cursor: return Cursor(self.client) diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/dbapi/cursor.py b/contrib/python/clickhouse-connect/clickhouse_connect/dbapi/cursor.py index af7f36f67b6..9b200aba3a3 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/dbapi/cursor.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/dbapi/cursor.py @@ -1,6 +1,7 @@ import logging import re from collections.abc import Mapping, Sequence +from typing import Any, cast from clickhouse_connect.datatypes.registry import get_from_name from clickhouse_connect.driver import Client @@ -23,34 +24,34 @@ class Cursor: def __init__(self, client: Client): self.client = client - self.arraysize = 1 + self.arraysize: int = 1 self.data: Sequence | None = None - self.names = [] - self.types = [] - self._rowcount = 0 - self._summary: list[dict[str, str]] = [] - self._ix = 0 + self.names: Sequence[str] = [] + self.types: Sequence[Any] = [] + self._rowcount: int = 0 + self._summary: list[dict[str, Any]] = [] + self._ix: int = 0 - def check_valid(self): + def check_valid(self) -> None: if self.data is None: raise ProgrammingError("Cursor is not valid") @property - def description(self): + def description(self) -> list[tuple[str, Any, None, None, None, None, bool]]: return [(n, t, None, None, None, None, True) for n, t in zip(self.names, self.types)] @property - def rowcount(self): + def rowcount(self) -> int: return self._rowcount @property - def summary(self) -> list[dict[str, str]]: + def summary(self) -> list[dict[str, Any]]: return self._summary - def close(self): + def close(self) -> None: self.data = None - def execute(self, operation: str, parameters=None): + def execute(self, operation: str, parameters: Any = None) -> None: if not parameters and isinstance(operation, str): # Per PEP 249 pyformat paramstyle, callers (e.g. SQLAlchemy) escape # literal percent signs as %% in operation strings. When there are @@ -80,7 +81,7 @@ class Cursor: self.names = meta_result.column_names self.types = [x.name for x in meta_result.column_types] - def _try_bulk_insert(self, operation: str, data): + def _try_bulk_insert(self, operation: str, data: Any) -> bool: match = insert_re.match(remove_sql_comments(operation)) if not match: return False @@ -94,24 +95,31 @@ class Cursor: op_columns = None if "VALUES" not in temp.upper(): return False + if not isinstance(data, Sequence) or len(data) == 0: + return False first_row = data[0] + col_names: list[str] | str + data_values: Sequence[Sequence[Any]] if isinstance(first_row, Mapping): - col_names = list(first_row.keys()) - if op_columns and {unescape_identifier(x) for x in op_columns} != set(col_names): + col_names = [str(k) for k in first_row.keys()] + if op_columns and {unescape_identifier(str(x)) for x in op_columns} != set(col_names): return False # Data sent in doesn't match the columns in the insert statement data_values = [list(row.values()) for row in data] elif isinstance(first_row, Sequence) and not isinstance(first_row, (str, bytes)): # PEP 249 also allows rows as sequences; take column names from the # insert statement if present, otherwise insert into all columns - col_names = [unescape_identifier(x) for x in op_columns] if op_columns else "*" + col_names = [unescape_identifier(str(x)) for x in op_columns] if op_columns else "*" data_values = data else: return False - self.client.insert(table, data_values, col_names) + insert_summary = self.client.insert(table, data_values, col_names) self.data = [] + self._rowcount = insert_summary.written_rows + self._ix = 0 + self._summary.append(insert_summary.summary) return True - def executemany(self, operation, parameters): + def executemany(self, operation: str, parameters: Any) -> None: if not parameters or self._try_bulk_insert(operation, parameters): return self.data = [] @@ -138,22 +146,25 @@ class Cursor: # Need to reset cursor _ix after performing an execute self._ix = 0 - def fetchall(self): + def fetchall(self) -> Sequence: self.check_valid() - ret = self.data[self._ix :] + data = cast(Sequence, self.data) + ret = data[self._ix :] self._ix = self._rowcount return ret - def fetchone(self): + def fetchone(self) -> Any: self.check_valid() if self._ix >= self._rowcount: return None - val = self.data[self._ix] + data = cast(Sequence, self.data) + val = data[self._ix] self._ix += 1 return val - def fetchmany(self, size: int = -1): + def fetchmany(self, size: int = -1) -> Sequence: self.check_valid() + data = cast(Sequence, self.data) if size < 0: # Fetch all remaining rows @@ -163,12 +174,12 @@ class Cursor: return [] end = min(self._ix + size, self._rowcount) - ret = self.data[self._ix : end] + ret = data[self._ix : end] self._ix = end return ret - def nextset(self): + def nextset(self) -> None: raise NotImplementedError - def callproc(self, *args, **kwargs): + def callproc(self, *args, **kwargs) -> None: raise NotImplementedError diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/__init__.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/__init__.py index 811286ad24c..1775a420230 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/__init__.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/__init__.py @@ -44,23 +44,24 @@ def _parse_connection_params( host: str | None, username: str | None, password: str, - port: int, - database: str, + port: int | None, + database: str | None, interface: str | None, secure: bool | str, dsn: str | None, kwargs: dict[str, Any], -) -> tuple[str, str | None, str, int, str, str]: +) -> tuple[str, str | None, str, int, str | None, str]: """Parse and normalize connection parameters including DSN parsing.""" + if database == "__default__": # legacy sentinel for "not specified" + database = None if dsn: parsed = urlparse(dsn) username = username or _unquote(parsed.username) password = password or _unquote(parsed.password) or "" host = host or parsed.hostname port = port or parsed.port - if parsed.path and (not database or database == "__default__"): - database = unquote(parsed.path[1:].split("/")[0]) - database = database or parsed.path + if not database and parsed.path: + database = unquote(parsed.path[1:].split("/")[0]) or None for k, v in parse_qs(parsed.query).items(): kwargs[k] = v[0] use_tls = str(secure).lower() == "true" or interface == "https" or (not interface and str(port) in ("443", "8443")) @@ -81,7 +82,12 @@ def _parse_connection_params( return host, username, password, port, database, interface -def _validate_access_token(access_token: str | None, token_provider: Callable[[], str] | None, username: str | None, password: str) -> None: +def _validate_access_token( + access_token: str | None, + token_provider: Callable[[], str | Awaitable[str]] | None, + username: str | None, + password: str, +) -> None: """Validate that token-based and username/password auth are not mixed.""" if (access_token or token_provider) and (username or password): raise ProgrammingError("Cannot use both token authentication and username/password") @@ -110,9 +116,9 @@ def create_client( password: str = "", access_token: str | None = None, token_provider: Callable[[], str] | None = None, - database: str = "__default__", + database: str | None = None, interface: str | None = None, - port: int = 0, + port: int | None = None, secure: bool | str = False, dsn: str | None = None, settings: dict[str, Any] | None = None, @@ -224,7 +230,7 @@ def create_client( interface, host, port, - username, + username or "", password, database, access_token, @@ -243,9 +249,9 @@ async def create_async_client( password: str = "", access_token: str | None = None, token_provider: Callable[[], str | Awaitable[str]] | None = None, - database: str = "__default__", + database: str | None = None, interface: str | None = None, - port: int = 0, + port: int | None = None, secure: bool | str = False, dsn: str | None = None, settings: dict[str, Any] | None = None, diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/asyncclient.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/asyncclient.py index b0301df84ca..d7bc03884d4 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/asyncclient.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/asyncclient.py @@ -14,17 +14,18 @@ import uuid import zlib import zoneinfo from base64 import b64encode -from collections.abc import Awaitable, Callable, Generator, Iterable, Sequence +from collections.abc import Awaitable, Callable, Generator, Sequence from datetime import timezone, tzinfo from importlib import import_module from importlib.metadata import version as dist_version -from typing import TYPE_CHECKING, Any, BinaryIO +from typing import TYPE_CHECKING, Any, BinaryIO, cast import aiohttp import lz4.frame import zstandard if TYPE_CHECKING: + import numpy import pandas import polars import pyarrow @@ -64,6 +65,7 @@ from clickhouse_connect.driver.query import ( from clickhouse_connect.driver.streaming import StreamingFileAdapter, StreamingInsertSource, StreamingResponseSource from clickhouse_connect.driver.summary import QuerySummary from clickhouse_connect.driver.transform import NativeTransform +from clickhouse_connect.driver.types import Closable logger = logging.getLogger(__name__) columns_only_re = re.compile(r"LIMIT 0\s*$", re.IGNORECASE) @@ -319,7 +321,7 @@ class AsyncClient(Client): self._ssl_context = ssl_context self._proxy_url = proxy_url - self._connector_kwargs = { + self._connector_kwargs: dict[str, Any] = { "limit": connector_limit, "limit_per_host": connector_limit_per_host, "keepalive_timeout": keepalive_timeout, @@ -337,10 +339,10 @@ class AsyncClient(Client): self._read_format = "Native" self._write_format = "Native" self._transform = NativeTransform() - self._client_settings = {} + self._client_settings: dict[str, str] = {} self._initialized = False - self._reported_libs = set() - self._last_pool_reset = None + self._reported_libs: set[str] = set() + self._last_pool_reset: float | None = None self.headers["User-Agent"] = self.headers["User-Agent"].replace("mode:sync;", "mode:async;") if headers: self.headers.update(headers) @@ -505,18 +507,18 @@ class AsyncClient(Client): self._session = None raise - async def __aenter__(self): + async def __aenter__(self) -> AsyncClient: """Async context manager entry.""" if not self._initialized: await self._initialize() return self - async def __aexit__(self, exc_type, exc_val, exc_tb): + async def __aexit__(self, exc_type, exc_val, exc_tb) -> bool: """Async context manager exit.""" await self.close() return False - async def close(self): # type: ignore[override] + async def close(self) -> None: # type: ignore[override] async with self._session_lock: old_lease = self._session_lease self._session_lease = None @@ -524,7 +526,7 @@ class AsyncClient(Client): await old_lease.wait_drained() await old_lease.session.close() - async def close_connections(self): # type: ignore[override] + async def close_connections(self) -> None: # type: ignore[override] """Rotate the connection pool: new requests use a fresh session; in-flight requests keep using the old session until they complete, then it's closed.""" async with self._session_lock: @@ -543,7 +545,7 @@ class AsyncClient(Client): await old_lease.wait_drained() await old_lease.session.close() - def set_client_setting(self, key, value): + def set_client_setting(self, key: str, value: Any) -> None: str_value = self._validate_setting(key, value, common.get_setting("invalid_setting_action")) if str_value is not None: self._client_settings[key] = str_value @@ -555,12 +557,12 @@ class AsyncClient(Client): # Run sync providers off the event loop; await async providers. # The provider may be called concurrently if multiple requests get a 516 at the same time; # it must be safe to invoke in parallel (e.g. if it hits an IdP, consider rate limiting). - result = await asyncio.get_running_loop().run_in_executor(None, self._token_provider) + result = await asyncio.get_running_loop().run_in_executor(None, cast(Callable[[], str | Awaitable[str]], self._token_provider)) if inspect.isawaitable(result): result = await result return result - def set_access_token(self, access_token: str): + def set_access_token(self, access_token: str) -> None: auth_header = self.headers.get("Authorization") if auth_header and not auth_header.startswith("Bearer"): raise ProgrammingError("Cannot set access token when a different auth type is used") @@ -578,17 +580,18 @@ class AsyncClient(Client): return final_query + fmt async def _query_with_context(self, context: QueryContext) -> QueryResult: # type: ignore[override] - headers = {} - params = {} + headers: dict[str, Any] = {} + params: dict[str, str] = {} if self.database: params["database"] = self.database if self.protocol_version: - params["client_protocol_version"] = self.protocol_version + params["client_protocol_version"] = str(self.protocol_version) context.block_info = True params.update(self._validate_settings(context.settings)) context.rename_response_column = self._rename_response_column use_form = use_form_encoding(context.final_query, context.bind_params, self.form_encode_query_params) + files: dict[str, Any] | None = None if not context.is_insert and columns_only_re.search(context.uncommented_query): fmt_json_query = f"{context.final_query}\n FORMAT JSON" fields = {"query": fmt_json_query} @@ -641,7 +644,7 @@ class AsyncClient(Client): logger.debug("Failed to rename col '%s'. Skipping rename. Error: %s", name, e) names.append(name) types.append(get_from_name(col["type"])) - return QueryResult([], None, tuple(names), tuple(types)) + return QueryResult([], None, tuple(names), tuple(types)) # type: ignore[arg-type] if self.compression: headers["Accept-Encoding"] = self.compression @@ -651,7 +654,7 @@ class AsyncClient(Client): final_query = self._prep_query(context) files = None - data = None + data: Any = None if use_form: fields = {"query": final_query} @@ -755,7 +758,10 @@ class AsyncClient(Client): """ if query and query.lower().strip().startswith("select __connect_version__"): return QueryResult( - [[f"ClickHouse Connect v.{common.version()} ⓒ ClickHouse Inc."]], None, ("connect_version",), (get_from_name("String"),) + [[f"ClickHouse Connect v.{common.version()} ⓒ ClickHouse Inc."]], + None, # type: ignore[arg-type] # QueryContext.generator not yet Optional; widen after #805 merges + ("connect_version",), + (get_from_name("String"),), # type: ignore[arg-type] ) if not context: context = self.create_query_context( @@ -856,7 +862,7 @@ class AsyncClient(Client): """ return (await self._context_query(locals(), use_numpy=False, streaming=True)).rows_stream - async def query_np( + async def query_np( # type: ignore[override] self, query: str | None = None, parameters: Sequence | dict[str, Any] | None = None, @@ -869,7 +875,7 @@ class AsyncClient(Client): context: QueryContext | None = None, external_data: ExternalData | None = None, transport_settings: dict[str, str] | None = None, - ): + ) -> numpy.ndarray: check_numpy() self._add_integration_tag("numpy") return (await self._context_query(locals(), use_numpy=True)).np_result @@ -910,7 +916,7 @@ class AsyncClient(Client): use_extended_dtypes: bool | None = None, transport_settings: dict[str, str] | None = None, tz_mode: TzMode | None = None, - ): + ) -> pandas.DataFrame: check_pandas() self._add_integration_tag("pandas") return (await self._context_query(locals(), use_numpy=True, as_pandas=True)).df_result @@ -938,7 +944,7 @@ class AsyncClient(Client): self._add_integration_tag("pandas") return (await self._context_query(locals(), use_numpy=True, as_pandas=True, streaming=True)).df_stream - async def _context_query(self, lcls: dict, **overrides): # type: ignore[override] + async def _context_query(self, lcls: dict, **overrides): """ Helper method to create query context and execute query. Matches sync client pattern for consistency. @@ -983,6 +989,8 @@ class AsyncClient(Client): raise ProgrammingError("Command sent without query or recognized data") from None if payload or files: + if isinstance(cmd, bytes): + raise ProgrammingError("Binary parameter bind cannot be combined with command data or external data") from None params["query"] = cmd else: payload = cmd @@ -1104,7 +1112,7 @@ class AsyncClient(Client): async for chunk in response.content.iter_any(): yield chunk - class _RawStreamSource: + class _RawStreamSource(Closable): def close(self): try: response.close() @@ -1164,7 +1172,7 @@ class AsyncClient(Client): self, table: str | None = None, data: Sequence[Sequence[Any]] | None = None, - column_names: str | Iterable[str] = "*", + column_names: str | Sequence[str] | None = "*", database: str | None = None, column_types: Sequence[ClickHouseType] | None = None, column_type_names: Sequence[str] | None = None, @@ -1195,6 +1203,8 @@ class AsyncClient(Client): if (context is None or context.empty) and data is None: raise ProgrammingError("No data specified for insert") from None if context is None: + if table is None: + raise ProgrammingError("No table specified for insert") from None context = await self.create_insert_context( table, column_names, @@ -1219,7 +1229,7 @@ class AsyncClient(Client): use_strings: bool | None = None, external_data: ExternalData | None = None, transport_settings: dict[str, str] | None = None, - ): + ) -> pyarrow.Table: """ Query method using the ClickHouse Arrow format to return a PyArrow table :param query: Query statement/format string @@ -1312,9 +1322,9 @@ class AsyncClient(Client): streaming_source = StreamingResponseSource(response, encoding=encoding, exception_tag=exception_tag) await streaming_source.start_producer(loop) - queue = AsyncSyncQueue(maxsize=10) + queue: AsyncSyncQueue = AsyncSyncQueue(maxsize=10) - class _ArrowStreamSource: + class _ArrowStreamSource(Closable): def __init__(self, source, q): self._source = source self._queue = q @@ -1404,7 +1414,7 @@ class AsyncClient(Client): check_polars() self._add_integration_tag("polars") - def converter(table: pyarrow.Table) -> polars.DataFrame: + def converter(table: pyarrow.Table) -> polars.DataFrame: # type: ignore[misc] table = _apply_arrow_tz_policy(table, self.tz_mode) return options.pl.from_arrow(table) @@ -1458,7 +1468,7 @@ class AsyncClient(Client): check_polars() self._add_integration_tag("polars") - def converter(table: pyarrow.Table) -> polars.DataFrame: + def converter(table: pyarrow.Table) -> polars.DataFrame: # type: ignore[misc] table = _apply_arrow_tz_policy(table, self.tz_mode) return options.pl.from_arrow(table) @@ -1482,9 +1492,9 @@ class AsyncClient(Client): streaming_source = StreamingResponseSource(response, encoding=encoding, exception_tag=exception_tag) await streaming_source.start_producer(loop) - queue = AsyncSyncQueue(maxsize=10) + queue: AsyncSyncQueue = AsyncSyncQueue(maxsize=10) - class _ArrowDFStreamSource: + class _ArrowDFStreamSource(Closable): def __init__(self, source, q): self._source = source self._queue = q @@ -1560,7 +1570,7 @@ class AsyncClient(Client): column_names, insert_block = arrow_buffer(arrow_table, compression) if hasattr(insert_block, "to_pybytes"): insert_block = insert_block.to_pybytes() - return await self.raw_insert(full_table, column_names, insert_block, settings, "Arrow", transport_settings) + return await self.raw_insert(full_table, column_names, insert_block, settings, "Arrow", transport_settings=transport_settings) async def insert_df_arrow( # type: ignore[override] self, @@ -1719,8 +1729,8 @@ class AsyncClient(Client): active_source.start_producer() return active_source.async_generator() - headers = {"Content-Type": "application/octet-stream"} - if context.compression: + headers: dict[str, Any] = {"Content-Type": "application/octet-stream"} + if isinstance(context.compression, str): headers["Content-Encoding"] = context.compression params = {} @@ -1829,9 +1839,11 @@ class AsyncClient(Client): query = f"INSERT INTO {table}{cols} FORMAT {fmt_str}" if not compression and isinstance(insert_block, str): insert_block = query + "\n" + insert_block - elif not compression and isinstance(insert_block, (bytes, bytearray, BinaryIO)): + elif not compression and isinstance(insert_block, (bytes, bytearray)): insert_block = (query + "\n").encode() + insert_block else: + # Generators, file-like objects, and compressed data: send the + # INSERT query as a URL param and stream the body as-is. params["query"] = query if self.database: @@ -2040,7 +2052,7 @@ class AsyncClient(Client): response = await session.request(**request_kwargs) if 200 <= response.status < 300 and not response.headers.get(ex_header): # Caller releases lease after consuming the body. - response._lease_release = _one_shot(lease.release) + response._lease_release = _one_shot(lease.release) # type: ignore[attr-defined] lease_released = True return response diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/asyncqueue.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/asyncqueue.py index a318f59cea1..3d1c277ebd6 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/asyncqueue.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/asyncqueue.py @@ -1,7 +1,7 @@ import asyncio import threading from collections import deque -from typing import Generic, TypeVar +from typing import Any, Generic, TypeVar from clickhouse_connect.driver.exceptions import ProgrammingError @@ -9,7 +9,8 @@ __all__ = ["AsyncSyncQueue", "Empty", "Full", "EOF_SENTINEL"] T = TypeVar("T") -EOF_SENTINEL = object() +# Typed Any so it can stand in for a queued T value on EOF without a cast in every reader. +EOF_SENTINEL: Any = object() class AsyncSyncQueue(Generic[T]): @@ -73,7 +74,8 @@ class AsyncSyncQueue(Generic[T]): while waiter_queue: fut = waiter_queue.popleft() if not fut.done(): - self._loop.call_soon_threadsafe(self._safe_set_result, fut) + # _bind_loop() runs before any Future is created, so _loop is always set here + self._loop.call_soon_threadsafe(self._safe_set_result, fut) # type: ignore[union-attr] break def shutdown(self): @@ -161,7 +163,8 @@ class _AsyncQueueInterface(Generic[T]): if self._p._shutdown: return EOF_SENTINEL - fut = self._p._loop.create_future() + # _bind_loop() is called at the top of get(), so _loop is always set here + fut = self._p._loop.create_future() # type: ignore[union-attr] self._p._async_getters.append(fut) try: @@ -185,7 +188,8 @@ class _AsyncQueueInterface(Generic[T]): self._p._wakeup_async_waiter(self._p._async_getters) return - fut = self._p._loop.create_future() + # _bind_loop() is called at the top of put(), so _loop is always set here + fut = self._p._loop.create_future() # type: ignore[union-attr] self._p._async_putters.append(fut) try: diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/binding.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/binding.py index b4f848dc77a..52fe6333ce7 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/binding.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/binding.py @@ -23,7 +23,7 @@ class DT64Param: def __init__(self, value: datetime): self.value = value - def format(self, tz: tzinfo, top_level: bool) -> str: + def format(self, tz: tzinfo | None, top_level: bool) -> str: value = self.value if tz: value = value.astimezone(tz) @@ -128,12 +128,13 @@ def bind_query( query: str, parameters: Sequence | dict[str, Any] | None, server_tz: tzinfo | None = None, -) -> tuple[str, dict[str, str]]: +) -> tuple[str | bytes, dict[str, str]]: query = query.rstrip(";") if not parameters: return query, {} binary_binds = None + bound_params: dict[str, str] = {} if isinstance(parameters, dict): params_copy = dict_copy(parameters) @@ -186,13 +187,14 @@ def bind_query( break binary_indexes[item_index + len(key)] = key, v item_index += len(key) - query = b"" + binary_out = b"" start = 0 for loc in sorted(binary_indexes.keys()): key, value = binary_indexes[loc] - query += binary_query[start:loc] + value + key + binary_out += binary_query[start:loc] + value + key start = loc - query += binary_query[start:] + binary_out += binary_query[start:] + return binary_out, bound_params return query, bound_params @@ -232,7 +234,7 @@ def escape_bytes(value): return "".join(f"{BS}x{b:02x}" for b in value) -def format_query_value(value: Any, server_tz: tzinfo = timezone.utc): +def format_query_value(value: Any, server_tz: tzinfo | None = timezone.utc): """ Format Python values in a ClickHouse query :param value: Python object @@ -269,11 +271,11 @@ def format_query_value(value: Any, server_tz: tzinfo = timezone.utc): return value -def str_query_value(value: Any, server_tz: tzinfo = timezone.utc): +def str_query_value(value: Any, server_tz: tzinfo | None = timezone.utc): return str(format_query_value(value, server_tz)) -def format_bind_value(value: Any, server_tz: tzinfo = timezone.utc, top_level: bool = True): +def format_bind_value(value: Any, server_tz: tzinfo | None = timezone.utc, top_level: bool = True): """ Format Python values in a ClickHouse query :param value: Python object diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/buffer.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/buffer.py index 97c01670b09..c0da348660f 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/buffer.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/buffer.py @@ -114,11 +114,11 @@ class ResponseBuffer(ByteSource): def read_str_col( self, num_rows: int, - encoding: str, + encoding: str | None, nullable: bool = False, null_obj: Any = None, ) -> Iterable[str]: - column = [] + column: list[Any] = [] app = column.append null_map = self.read_bytes(num_rows) if nullable else None for ix in range(num_rows): @@ -148,7 +148,7 @@ class ResponseBuffer(ByteSource): def read_fixed_str_col(self, sz: int, num_rows: int, encoding: str) -> Iterable[str]: source = self.read_bytes(sz * num_rows) - column = [] + column: list[str] = [] app = column.append for ix in range(0, sz * num_rows, sz): try: @@ -167,7 +167,7 @@ class ResponseBuffer(ByteSource): return column @property - def last_message(self) -> bytes: + def last_message(self) -> bytes | None: # type: ignore[override] # overrides writable attr with property return self.buffer def close(self): diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/client.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/client.py index e5da4ec5736..46ce2b4b76a 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/client.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/client.py @@ -3,12 +3,13 @@ from __future__ import annotations import io import logging from abc import ABC, abstractmethod -from collections.abc import Generator, Iterable, Sequence +from collections.abc import Generator, Sequence from datetime import timezone, tzinfo from typing import ( TYPE_CHECKING, Any, BinaryIO, + cast, ) from zoneinfo import ZoneInfoNotFoundError @@ -55,6 +56,7 @@ from clickhouse_connect.driver.query import ( to_arrow_batches, ) from clickhouse_connect.driver.summary import QuerySummary +from clickhouse_connect.driver.types import Closable if TYPE_CHECKING: import numpy @@ -62,7 +64,7 @@ if TYPE_CHECKING: import polars import pyarrow -io.DEFAULT_BUFFER_SIZE = 1024 * 256 +io.DEFAULT_BUFFER_SIZE = 1024 * 256 # type: ignore[misc] # override module default buffer size logger = logging.getLogger(__name__) arrow_str_setting = "output_format_arrow_string_as_string" @@ -117,8 +119,8 @@ class Client(ABC): compression: str | None = None write_compression: str | None = None protocol_version = 0 - valid_transport_settings = set() - optional_transport_settings = set() + valid_transport_settings: set[str] = set() + optional_transport_settings: set[str] = set() database = None max_error_message = 0 _tz_source: TzSource = "auto" @@ -169,7 +171,7 @@ class Client(ABC): """ self.query_limit = coerce_int(query_limit) self.query_retries = coerce_int(query_retries) - if database and not database == "__default__": + if database and database != "__default__": self.database = database if show_clickhouse_errors is not None: self.show_clickhouse_errors = coerce_bool(show_clickhouse_errors) @@ -184,9 +186,9 @@ class Client(ABC): self._tz_source = resolved_tz_source # Initialize attributes that will be set during connection - self.server_version = None - self.server_tz = timezone.utc - self.server_settings = {} + self.server_version: str | None = None + self.server_tz: tzinfo = timezone.utc + self.server_settings: dict[str, SettingDef] = {} if autoconnect: self._init_common_settings(resolved_tz_source) @@ -196,7 +198,10 @@ class Client(ABC): def _init_common_settings(self, tz_source: TzSource): self.server_tz, self._dst_safe = timezone.utc, True - self.server_version, server_tz = tuple(self.command("SELECT version(), timezone()", use_database=False)) + version_result = self.command("SELECT version(), timezone()", use_database=False) + if not isinstance(version_result, Sequence) or isinstance(version_result, str): + raise OperationalError(f"Unexpected response to server version query: {version_result!r}") + self.server_version, server_tz = version_result[0], version_result[1] try: server_tz_info = tzutil.resolve_zone(server_tz) server_tz_info, self._dst_safe = tzutil.normalize_timezone(server_tz_info, trust_fixed_offset=True) @@ -247,8 +252,10 @@ class Client(ABC): :param settings: Dictionary of setting name and values :return: A filtered dictionary of settings with values rendered as strings """ - validated = {} + validated: dict[str, str] = {} invalid_action = common.get_setting("invalid_setting_action") + if not settings: + return validated for key, value in settings.items(): str_value = self._validate_setting(key, value, invalid_action) if str_value is not None: @@ -349,7 +356,7 @@ class Client(ABC): column_oriented: bool | None = None, use_numpy: bool | None = None, max_str_len: int | None = None, - context: QueryContext = None, + context: QueryContext | None = None, query_tz: str | tzinfo | None = None, column_tzs: dict[str, str | tzinfo] | None = None, external_data: ExternalData | None = None, @@ -363,14 +370,17 @@ class Client(ABC): """ if query and query.lower().strip().startswith("select __connect_version__"): return QueryResult( - [[f"ClickHouse Connect v.{version()} ⓒ ClickHouse Inc."]], None, ("connect_version",), (get_from_name("String"),) + [[f"ClickHouse Connect v.{version()} ⓒ ClickHouse Inc."]], + None, + ("connect_version",), + (get_from_name("String"),), ) kwargs = locals().copy() del kwargs["self"] query_context = self.create_query_context(**kwargs) if query_context.is_command: response = self.command( - query, + cast(str, query), parameters=query_context.parameters, settings=query_context.settings, external_data=query_context.external_data, @@ -390,7 +400,7 @@ class Client(ABC): column_formats: dict[str, str | dict[str, str]] | None = None, encoding: str | None = None, use_none: bool | None = None, - context: QueryContext = None, + context: QueryContext | None = None, query_tz: str | tzinfo | None = None, column_tzs: dict[str, str | tzinfo] | None = None, external_data: ExternalData | None = None, @@ -413,7 +423,7 @@ class Client(ABC): column_formats: dict[str, str | dict[str, str]] | None = None, encoding: str | None = None, use_none: bool | None = None, - context: QueryContext = None, + context: QueryContext | None = None, query_tz: str | tzinfo | None = None, column_tzs: dict[str, str | tzinfo] | None = None, external_data: ExternalData | None = None, @@ -436,7 +446,7 @@ class Client(ABC): column_formats: dict[str, str | dict[str, str]] | None = None, encoding: str | None = None, use_none: bool | None = None, - context: QueryContext = None, + context: QueryContext | None = None, query_tz: str | tzinfo | None = None, column_tzs: dict[str, str | tzinfo] | None = None, external_data: ExternalData | None = None, @@ -456,7 +466,7 @@ class Client(ABC): query: str, parameters: Sequence | dict[str, Any] | None = None, settings: dict[str, Any] | None = None, - fmt: str = None, + fmt: str | None = None, use_database: bool = True, external_data: ExternalData | None = None, transport_settings: dict[str, str] | None = None, @@ -480,7 +490,7 @@ class Client(ABC): query: str, parameters: Sequence | dict[str, Any] | None = None, settings: dict[str, Any] | None = None, - fmt: str = None, + fmt: str | None = None, use_database: bool = True, external_data: ExternalData | None = None, transport_settings: dict[str, str] | None = None, @@ -508,7 +518,7 @@ class Client(ABC): encoding: str | None = None, use_none: bool | None = None, max_str_len: int | None = None, - context: QueryContext = None, + context: QueryContext | None = None, external_data: ExternalData | None = None, transport_settings: dict[str, str] | None = None, ) -> numpy.ndarray: @@ -531,7 +541,7 @@ class Client(ABC): encoding: str | None = None, use_none: bool | None = None, max_str_len: int | None = None, - context: QueryContext = None, + context: QueryContext | None = None, external_data: ExternalData | None = None, transport_settings: dict[str, str] | None = None, ) -> StreamContext: @@ -557,7 +567,7 @@ class Client(ABC): use_na_values: bool | None = None, query_tz: str | None = None, column_tzs: dict[str, str | tzinfo] | None = None, - context: QueryContext = None, + context: QueryContext | None = None, external_data: ExternalData | None = None, use_extended_dtypes: bool | None = None, transport_settings: dict[str, str] | None = None, @@ -585,7 +595,7 @@ class Client(ABC): use_na_values: bool | None = None, query_tz: str | None = None, column_tzs: dict[str, str | tzinfo] | None = None, - context: QueryContext = None, + context: QueryContext | None = None, external_data: ExternalData | None = None, use_extended_dtypes: bool | None = None, transport_settings: dict[str, str] | None = None, @@ -688,7 +698,7 @@ class Client(ABC): if as_pandas and use_extended_dtypes is None: use_extended_dtypes = True return QueryContext( - query=query, + query=cast(str | bytes, query), parameters=parameters, settings=settings, query_formats=query_formats, @@ -766,13 +776,16 @@ class Client(ABC): self._add_integration_tag("arrow") settings = self._update_arrow_settings(settings, use_strings) return to_arrow_batches( - self.raw_stream( - query, - parameters, - settings, - fmt="ArrowStream", - external_data=external_data, - transport_settings=transport_settings, + cast( + io.IOBase, + self.raw_stream( + query, + parameters, + settings, + fmt="ArrowStream", + external_data=external_data, + transport_settings=transport_settings, + ), ) ) @@ -814,7 +827,7 @@ class Client(ABC): check_polars() self._add_integration_tag("polars") - def converter(table: pyarrow.Table) -> polars.DataFrame: + def converter(table: pyarrow.Table) -> polars.DataFrame: # type: ignore[misc] table = _apply_arrow_tz_policy(table, self.tz_mode) return options.pl.from_arrow(table) @@ -867,7 +880,7 @@ class Client(ABC): check_polars() self._add_integration_tag("polars") - def converter(table: pyarrow.Table) -> polars.DataFrame: + def converter(table: pyarrow.Table) -> polars.DataFrame: # type: ignore[misc] table = _apply_arrow_tz_policy(table, self.tz_mode) return options.pl.from_arrow(table) else: @@ -882,7 +895,7 @@ class Client(ABC): for batch in reader: yield converter(batch) - return StreamContext(raw_stream, df_generator()) + return StreamContext(cast(Closable, raw_stream), df_generator()) def _update_arrow_settings(self, settings: dict[str, Any] | None, use_strings: bool | None) -> dict[str, Any]: settings = dict_copy(settings) @@ -903,8 +916,8 @@ class Client(ABC): self, cmd: str, parameters: Sequence | dict[str, Any] | None = None, - data: str | bytes = None, - settings: dict[str, Any] = None, + data: str | bytes | None = None, + settings: dict[str, Any] | None = None, use_database: bool = True, external_data: ExternalData | None = None, transport_settings: dict[str, str] | None = None, @@ -934,14 +947,14 @@ class Client(ABC): def insert( self, table: str | None = None, - data: Sequence[Sequence[Any]] = None, - column_names: str | Iterable[str] = "*", + data: Sequence[Sequence[Any]] | None = None, + column_names: str | Sequence[str] | None = "*", database: str | None = None, - column_types: Sequence[ClickHouseType] = None, - column_type_names: Sequence[str] = None, + column_types: Sequence[ClickHouseType] | None = None, + column_type_names: Sequence[str] | None = None, column_oriented: bool = False, settings: dict[str, Any] | None = None, - context: InsertContext = None, + context: InsertContext | None = None, transport_settings: dict[str, str] | None = None, ) -> QuerySummary: """ @@ -966,6 +979,8 @@ class Client(ABC): if (context is None or context.empty) and data is None: raise ProgrammingError("No data specified for insert") from None if context is None: + if table is None: + raise ProgrammingError("No table specified for insert") from None context = self.create_insert_context( table, column_names, @@ -984,14 +999,14 @@ class Client(ABC): def insert_df( self, - table: str = None, + table: str | None = None, df=None, database: str | None = None, settings: dict | None = None, column_names: Sequence[str] | None = None, - column_types: Sequence[ClickHouseType] = None, - column_type_names: Sequence[str] = None, - context: InsertContext = None, + column_types: Sequence[ClickHouseType] | None = None, + column_type_names: Sequence[str] | None = None, + context: InsertContext | None = None, transport_settings: dict[str, str] | None = None, ) -> QuerySummary: """ @@ -1034,7 +1049,7 @@ class Client(ABC): self, table: str, arrow_table, - database: str = None, + database: str | None = None, settings: dict | None = None, transport_settings: dict[str, str] | None = None, ) -> QuerySummary: @@ -1051,7 +1066,7 @@ class Client(ABC): full_table = table if "." in table or not database else f"{database}.{table}" compression = self.write_compression if self.write_compression in ("zstd", "lz4") else None column_names, insert_block = arrow_buffer(arrow_table, compression) - return self.raw_insert(full_table, column_names, insert_block, settings, "Arrow", transport_settings) + return self.raw_insert(full_table, column_names, insert_block, settings, "Arrow", transport_settings=transport_settings) def insert_df_arrow( self, @@ -1117,8 +1132,8 @@ class Client(ABC): table: str, column_names: str | Sequence[str] | None = None, database: str | None = None, - column_types: Sequence[ClickHouseType] = None, - column_type_names: Sequence[str] = None, + column_types: Sequence[ClickHouseType] | None = None, + column_type_names: Sequence[str] | None = None, column_oriented: bool = False, settings: dict[str, Any] | None = None, data: Sequence[Sequence[Any]] | None = None, @@ -1146,7 +1161,7 @@ class Client(ABC): full_table = f"{quote_identifier(database)}.{quote_identifier(table)}" else: full_table = quote_identifier(table) - column_defs = [] + column_defs: list[ColumnDef] = [] if column_types is None and column_type_names is None: describe_result = self.query(f"DESCRIBE TABLE {full_table}", settings=settings) column_defs = [ @@ -1189,7 +1204,7 @@ class Client(ABC): :return: True if version_str is greater than the server_version, False if less than """ try: - server_parts = [int(x) for x in self.server_version.split(".") if x.isnumeric()] + server_parts = [int(x) for x in (self.server_version or "").split(".") if x.isnumeric()] server_parts.extend([0] * (4 - len(server_parts))) version_parts = [int(x) for x in version_str.split(".")] version_parts.extend([0] * (4 - len(version_parts))) @@ -1222,7 +1237,7 @@ class Client(ABC): self, table: str, column_names: Sequence[str] | None = None, - insert_block: str | bytes | Generator[bytes, None, None] | BinaryIO = None, + insert_block: str | bytes | Generator[bytes, None, None] | BinaryIO | None = None, settings: dict | None = None, fmt: str | None = None, compression: str | None = None, diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/common.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/common.py index a4acbd292df..fc609bfd84b 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/common.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/common.py @@ -3,6 +3,7 @@ import asyncio import struct import sys from collections.abc import Callable, Generator, MutableSequence, Sequence +from io import IOBase from typing import Any from clickhouse_connect.driver.exceptions import DataError, ProgrammingError, StreamClosedError @@ -107,7 +108,7 @@ def unescape_identifier(x: str) -> str: return x -def dict_copy(source: dict = None, update: dict | None = None) -> dict: +def dict_copy(source: dict | None = None, update: dict | None = None) -> dict: copy = source.copy() if source else {} if update: copy.update(update) @@ -153,10 +154,13 @@ class SliceView(Sequence): slots = ("_source", "_range") + _source: Sequence + _range: range + def __init__(self, source: Sequence, source_slice: slice | None = None): if isinstance(source, SliceView): self._source = source._source - self._range = source._range[source_slice] + self._range = source._range if source_slice is None else source._range[source_slice] else: self._source = source if source_slice is None: @@ -200,7 +204,7 @@ class StreamContext: __slots__ = "source", "gen", "_in_context" - def __init__(self, source: Closable, gen: Generator): + def __init__(self, source: Closable | IOBase, gen: Generator): self.source = source self.gen = gen self._in_context = False diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/compression.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/compression.py index 802b6e815c2..b6533a95711 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/compression.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/compression.py @@ -1,5 +1,4 @@ import zlib -from abc import abstractmethod import lz4 import lz4.frame @@ -17,14 +16,13 @@ if brotli: available_compression.append("br") available_compression.extend(["gzip", "deflate"]) -comp_map = {} +comp_map: dict[str, "Compressor | type[Compressor]"] = {} class Compressor: def __init_subclass__(cls, tag: str, thread_safe: bool = True): comp_map[tag] = cls() if thread_safe else cls - @abstractmethod def compress_block(self, block) -> bytes | bytearray: return block @@ -66,11 +64,10 @@ class BrotliCompressor(Compressor, tag="br"): null_compressor = Compressor() -def get_compressor(compression: str) -> Compressor: +def get_compressor(compression: str | None) -> Compressor: if not compression: return null_compressor comp = comp_map[compression] - try: - return comp() - except TypeError: + if isinstance(comp, Compressor): return comp + return comp() diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/context.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/context.py index 50b8315a0db..eec0ed517ed 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/context.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/context.py @@ -5,7 +5,7 @@ from typing import Any logger = logging.getLogger(__name__) -_empty_map = {} +_empty_map: dict[Any, Any] = {} class BaseQueryContext: @@ -38,7 +38,7 @@ class BaseQueryContext: self.query_formats = query_formats or {} self.column_formats = column_formats or {} self.transport_settings = transport_settings - self.column_name = None + self.column_name: str | None = None self.encoding = encoding self.use_numpy = use_numpy self.use_extended_dtypes = use_extended_dtypes diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/dataconv.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/dataconv.py index 5a6d1786f8a..9de76e3fbdd 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/dataconv.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/dataconv.py @@ -17,11 +17,14 @@ MONTH_DAYS_LEAP = (0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366) def read_ipv4_col(source: ByteSource, num_rows: int): column = source.read_array("I", num_rows) fast_ip_v4 = IPv4Address.__new__ - new_col = [] + new_col: list[IPv4Address] = [] app = new_col.append for x in column: ipv4 = fast_ip_v4(IPv4Address) - ipv4._ip = x + # _ip is CPython's private backing int for the address. + # It's directly set to bypass IPv4Address.__init__ + # for speed when bulk-decoding a column + ipv4._ip = x # type: ignore[attr-defined] app(ipv4) return new_col @@ -103,7 +106,7 @@ def read_uuid_col(source: ByteSource, num_rows: int): new_uuid = UUID.__new__ unsafe = SafeUUID.unsafe oset = object.__setattr__ - column = [] + column: list[UUID] = [] app = column.append for i in range(num_rows): ix = i << 1 @@ -173,7 +176,7 @@ def write_str_col(column: Sequence, nullable: bool, encoding: str | None, dest: return 0 -def write_native_col(code: str, column: Sequence, dest: bytearray, col_name: object = None) -> int: +def write_native_col(code: str, column: Sequence, dest: bytearray, col_name: str | None = None) -> int: """ Pure Python fallback for write_native_col. Delegates to write_array which uses struct.pack. diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/ddl.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/ddl.py index d6137d124b4..b682fbfa4e6 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/ddl.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/ddl.py @@ -1,24 +1,30 @@ from __future__ import annotations from collections.abc import Sequence -from typing import TYPE_CHECKING, NamedTuple +from typing import TYPE_CHECKING, NamedTuple, Protocol -from clickhouse_connect.datatypes.base import ClickHouseType from clickhouse_connect.driver.options import check_arrow if TYPE_CHECKING: import pyarrow +class _NamedType(Protocol): + """Structural type for anything exposing a ClickHouse type ``name``.""" + + @property + def name(self) -> str: ... + + class TableColumnDef(NamedTuple): """ Simplified ClickHouse Table Column definition for DDL """ name: str - ch_type: ClickHouseType - expr_type: str = None - expr: str = None + ch_type: _NamedType + expr_type: str | None = None + expr: str | None = None @property def col_expr(self): diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/exceptions.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/exceptions.py index cddcac55f59..f00ff9aa712 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/exceptions.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/exceptions.py @@ -29,7 +29,7 @@ class ClickHouseError(Exception): """Exception related to operation with ClickHouse.""" -class Warning(Warning, ClickHouseError): # noqa: N818 +class Warning(Warning, ClickHouseError): # type: ignore[misc] # noqa: N818 """Exception raised for important warnings like data truncations while inserting, etc.""" diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/httpclient.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/httpclient.py index 170c2ab6ab8..9d1aa98b0da 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/httpclient.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/httpclient.py @@ -8,7 +8,7 @@ from base64 import b64encode from collections.abc import Callable, Generator, Sequence from importlib import import_module from importlib.metadata import version as dist_version -from typing import Any, BinaryIO +from typing import Any, BinaryIO, cast from urllib.parse import urlencode from urllib3 import Timeout @@ -43,7 +43,7 @@ from clickhouse_connect.driver.httputil import ( get_response_data, ) from clickhouse_connect.driver.insert import InsertContext -from clickhouse_connect.driver.query import QueryContext, QueryResult, TzSource, returns_empty_string_on_empty_body +from clickhouse_connect.driver.query import QueryContext, QueryResult, TzMode, TzSource, returns_empty_string_on_empty_body from clickhouse_connect.driver.summary import QuerySummary from clickhouse_connect.driver.transform import NativeTransform @@ -57,7 +57,7 @@ _REMOTE_CLOSE_ERRORS = (ConnectionResetError, BrokenPipeError) class HttpClient(Client): - params = {} + params: dict[str, str] = {} valid_transport_settings = { "database", "buffer_size", @@ -84,7 +84,7 @@ class HttpClient(Client): port: int, username: str, password: str, - database: str, + database: str | None, access_token: str | None = None, token_provider: Callable[[], str] | None = None, compress: bool | str = True, @@ -141,7 +141,7 @@ class HttpClient(Client): self.headers["X-ClickHouse-SSL-Certificate-Auth"] = "on" if not self.http and (server_host_name or ca_cert or client_cert or not verify or https_proxy): - options = {"verify": verify} + options: dict[str, Any] = {"verify": verify} dict_add(options, "ca_cert", ca_cert) dict_add(options, "client_cert", client_cert) dict_add(options, "client_cert_key", client_cert_key) @@ -167,7 +167,7 @@ class HttpClient(Client): elif (not client_cert or tls_mode in ("strict", "proxy")) and username: self.headers["Authorization"] = "Basic " + b64encode(f"{username}:{password}".encode()).decode() - self._reported_libs = set() + self._reported_libs: set[str] = set() self.headers["User-Agent"] = common.build_client_name(client_name) if headers: self.headers.update(headers) @@ -184,7 +184,7 @@ class HttpClient(Client): self._send_progress = None self._send_comp_setting = False self._progress_interval = None - self._active_session = None + self._active_session: str | None = None self._rename_response_column = rename_response_column # allow to override the global autogenerate_session_id setting via the constructor params @@ -220,7 +220,7 @@ class HttpClient(Client): query_retries=query_retries, server_host_name=server_host_name, tz_source=tz_source, - tz_mode=tz_mode, + tz_mode=cast(TzMode | None, tz_mode), show_clickhouse_errors=show_clickhouse_errors, autoconnect=True, ) @@ -265,16 +265,17 @@ class HttpClient(Client): return final_query + fmt def _query_with_context(self, context: QueryContext) -> QueryResult: - headers = {} - params = {} + headers: dict[str, Any] = {} + params: dict[str, str] = {} if self.database: params["database"] = self.database if self.protocol_version: - params["client_protocol_version"] = self.protocol_version + params["client_protocol_version"] = str(self.protocol_version) context.block_info = True params.update(self._validate_settings(context.settings)) context.rename_response_column = self._rename_response_column use_form = use_form_encoding(context.final_query, context.bind_params, self.form_encode_query_params) + fields: dict[str, Any] | None = None if not context.is_insert and columns_only_re.search(context.uncommented_query): # Mirror normal query behavior for form encoding and external data fmt_json_query = f"{context.final_query}\n FORMAT JSON" @@ -309,7 +310,7 @@ class HttpClient(Client): logger.debug("Failed to rename col '%s'. Skipping rename. Error: %s", name, e) names.append(name) types.append(registry.get_from_name(col["type"])) - return QueryResult([], None, tuple(names), tuple(types)) + return QueryResult([], None, tuple(names), tuple(types)) # type: ignore[arg-type] if self.compression: headers["Accept-Encoding"] = self.compression @@ -347,10 +348,12 @@ class HttpClient(Client): ) exception_tag = response.headers.get(ex_tag_header) byte_source = RespBuffCls(ResponseSource(response, exception_tag=exception_tag)) - context.set_response_tz(self._check_tz_change(response.headers.get("X-ClickHouse-Timezone"))) + response_tz = self._check_tz_change(response.headers.get("X-ClickHouse-Timezone")) + if response_tz is not None: + context.set_response_tz(response_tz) query_result = self._transform.parse_response(byte_source, context) query_result.summary = self._summary(response) - return query_result + return cast(QueryResult, query_result) def data_insert(self, context: InsertContext) -> QuerySummary: """ @@ -371,7 +374,7 @@ class HttpClient(Client): headers = {"Content-Type": "application/octet-stream"} if context.compression is None: context.compression = self.write_compression - if context.compression: + if isinstance(context.compression, str): headers["Content-Encoding"] = context.compression block_gen = self._transform.build_insert(context) @@ -401,9 +404,9 @@ class HttpClient(Client): def raw_insert( self, - table: str = None, + table: str | None = None, column_names: Sequence[str] | None = None, - insert_block: str | bytes | Generator[bytes, None, None] | BinaryIO = None, + insert_block: str | bytes | Generator[bytes, None, None] | BinaryIO | None = None, settings: dict | None = None, fmt: str | None = None, compression: str | None = None, @@ -421,9 +424,11 @@ class HttpClient(Client): query = f"INSERT INTO {table}{cols} FORMAT {fmt if fmt else self._write_format}" if not compression and isinstance(insert_block, str): insert_block = query + "\n" + insert_block - elif not compression and isinstance(insert_block, (bytes, bytearray, BinaryIO)): + elif not compression and isinstance(insert_block, (bytes, bytearray)): insert_block = (query + "\n").encode() + insert_block else: + # Generators, file-like objects, and compressed data: send the + # INSERT query as a URL param and stream the body as-is. params["query"] = query if self.database: params["database"] = self.database @@ -448,7 +453,7 @@ class HttpClient(Client): self, cmd: str, parameters: Sequence | dict[str, Any] | None = None, - data: str | bytes = None, + data: str | bytes | None = None, settings: dict | None = None, use_database: bool = True, external_data: ExternalData | None = None, @@ -457,9 +462,9 @@ class HttpClient(Client): """ See BaseClient doc_string for this method """ - cmd, params = bind_query(cmd, parameters, self.server_tz) - headers = {} - payload = None + bound_cmd, params = bind_query(cmd, parameters, self.server_tz) + headers: dict[str, Any] = {} + payload: str | bytes | None = None fields = None if external_data: if data: @@ -472,12 +477,14 @@ class HttpClient(Client): elif isinstance(data, bytes): headers["Content-Type"] = "application/octet-stream" payload = data - if payload is None and not cmd: + if payload is None and not bound_cmd: raise ProgrammingError("Command sent without query or recognized data") from None if payload or fields: - params["query"] = cmd + if isinstance(bound_cmd, bytes): + raise ProgrammingError("Binary parameter bind cannot be combined with command data or external data") from None + params["query"] = bound_cmd else: - payload = cmd + payload = bound_cmd if use_database and self.database: params["database"] = self.database params.update(self._validate_settings(settings or {})) @@ -495,7 +502,7 @@ class HttpClient(Client): return result except UnicodeDecodeError: return str(response.data) - if returns_empty_string_on_empty_body(cmd): + if returns_empty_string_on_empty_body(bound_cmd): return "" return QuerySummary(self._summary(response)) @@ -546,7 +553,7 @@ class HttpClient(Client): stream: bool = False, server_wait: bool = True, fields: dict[str, tuple] | None = None, - error_handler: Callable = None, + error_handler: Callable | None = None, retry_body: Callable[[], Any] | None = None, ) -> HTTPResponse: if isinstance(data, str): @@ -571,7 +578,7 @@ class HttpClient(Client): final_params["query_id"] = str(uuid.uuid4()) url = f"{self.url}?{urlencode(final_params)}" - kwargs = {"headers": headers, "timeout": self.timeout, "retries": self.http_retries, "preload_content": not stream} + kwargs: dict[str, Any] = {"headers": headers, "timeout": self.timeout, "retries": self.http_retries, "preload_content": not stream} if self.server_host_name: kwargs["assert_same_host"] = False kwargs["headers"].update({"Host": self.server_host_name}) @@ -579,7 +586,7 @@ class HttpClient(Client): kwargs["fields"] = fields else: kwargs["body"] = data - check_conn_expiration(self.http) + check_conn_expiration(cast(PoolManager, self.http)) query_session = final_params.get("session_id") while True: attempts += 1 @@ -593,7 +600,7 @@ class HttpClient(Client): # throw an error instead, but in most cases this more helpful error will be thrown first self._active_session = query_session try: - response = self.http.request(method, url, **kwargs) + response: HTTPResponse = cast(HTTPResponse, cast(PoolManager, self.http).request(method, url, **kwargs)) except HTTPError as ex: # Always allow at least one retry on a clean connection error so a single stale # keep-alive socket doesn't surface to the caller, and additionally honor the @@ -639,7 +646,7 @@ class HttpClient(Client): kwargs["body"] = retry_body() response.close() logger.debug("Refreshing access token after authentication failure") - elif error_handler: + elif error_handler is not None: error_handler(response) else: self._error_handler(response) @@ -649,7 +656,7 @@ class HttpClient(Client): query: str, parameters: Sequence | dict[str, Any] | None = None, settings: dict[str, Any] | None = None, - fmt: str = None, + fmt: str | None = None, use_database: bool = True, external_data: ExternalData | None = None, transport_settings: dict[str, str] | None = None, @@ -665,7 +672,7 @@ class HttpClient(Client): query: str, parameters: Sequence | dict[str, Any] | None = None, settings: dict[str, Any] | None = None, - fmt: str = None, + fmt: str | None = None, use_database: bool = True, external_data: ExternalData | None = None, transport_settings: dict[str, str] | None = None, @@ -689,7 +696,7 @@ class HttpClient(Client): query: str, parameters: Sequence | dict[str, Any] | None, settings: dict[str, Any] | None, - fmt: str, + fmt: str | None, use_database: bool, external_data: ExternalData | None, ): @@ -699,22 +706,24 @@ class HttpClient(Client): params = self._validate_settings(settings or {}) if use_database and self.database: params["database"] = self.database - fields = {} + form_fields: dict[str, Any] = {} + fields: dict[str, Any] | None = form_fields use_form = use_form_encoding(final_query, bind_params, self.form_encode_query_params) # Setup query body if external_data and not use_form and isinstance(final_query, bytes): raise ProgrammingError("Binary query cannot be placed in URL when using External Data; enable form encoding.") # Setup additional query parameters and body + body: str | bytes = b"" if use_form: - body = b"" - fields["query"] = final_query - fields.update(bind_params) + form_fields["query"] = final_query + form_fields.update(bind_params) if external_data: params.update(external_data.query_params) - fields.update(external_data.form_data) + form_fields.update(external_data.form_data) elif external_data: params.update(bind_params) - body = b"" + # Guaranteed str: the check above raises if external_data and not use_form and bytes + assert isinstance(final_query, str) params["query"] = final_query params.update(external_data.query_params) fields = external_data.form_data @@ -774,20 +783,20 @@ class HttpClient(Client): """ try: headers = dict_copy(self.headers) - kwargs = {"headers": headers, "timeout": 3, "preload_content": True} + kwargs: dict[str, Any] = {"headers": headers, "timeout": 3, "preload_content": True} if self.server_host_name: kwargs["assert_same_host"] = False headers["Host"] = self.server_host_name - response = self.http.request("GET", f"{self.url}/ping", **kwargs) + response = cast(PoolManager, self.http).request("GET", f"{self.url}/ping", **kwargs) return 200 <= response.status < 300 except HTTPError: logger.debug("ping failed", exc_info=True) return False def close_connections(self) -> None: - self.http.clear() + cast(PoolManager, self.http).clear() def close(self) -> None: if self._owns_pool_manager: - self.http.clear() - all_managers.pop(self.http, None) + cast(PoolManager, self.http).clear() + all_managers.pop(cast(PoolManager, self.http), None) diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/httputil.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/httputil.py index d135e94194d..0f7a6cfa091 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/httputil.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/httputil.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) # Increase this number just to be safe when ClickHouse is returning progress headers -http.client._MAXHEADERS = 10000 +http.client._MAXHEADERS = 10000 # type: ignore[attr-defined] DEFAULT_KEEP_INTERVAL = 30 DEFAULT_KEEP_COUNT = 3 @@ -42,8 +42,8 @@ core_socket_options = [ ] logging.getLogger("urllib3").setLevel(logging.WARNING) -_proxy_managers = {} -all_managers = {} +_proxy_managers: dict[str, PoolManager] = {} +all_managers: dict[PoolManager, int] = {} @atexit.register @@ -74,7 +74,7 @@ def get_pool_manager_options( if getattr(socket, "TCP_KEEPCNT", None) is not None: socket_options.append((SOCKET_TCP, socket.TCP_KEEPCNT, keep_count)) if getattr(socket, "TCP_KEEPIDLE", None) is not None: - socket_options.append((SOCKET_TCP, socket.TCP_KEEPIDLE, keep_idle)) + socket_options.append((SOCKET_TCP, socket.TCP_KEEPIDLE, keep_idle)) # type: ignore[attr-defined] if sys.platform == "darwin": socket_options.append((SOCKET_TCP, getattr(socket, "TCP_KEEPALIVE", 0x10), keep_interval)) options["maxsize"] = options.get("maxsize", 8) @@ -119,7 +119,7 @@ def get_pool_manager( raise ProgrammingError("Only one of http_proxy or https_proxy should be specified") if not http_proxy.startswith("http"): http_proxy = f"http://{http_proxy}" - manager = ProxyManager(http_proxy, **options) + manager: PoolManager = ProxyManager(http_proxy, **options) elif https_proxy: if not https_proxy.startswith("http"): https_proxy = f"https://{https_proxy}" @@ -238,7 +238,6 @@ class ResponseSource: done = False current_size = 0 read_gen = response.stream(chunk_size, decompress is None) - data_received = False read_error = None while True: while not done: @@ -246,7 +245,7 @@ class ResponseSource: try: chunk = next(read_gen, None) # Always try to read at least one chunk if there are any left except Exception as ex: - # Store the exception for potential re-raising if no data was received + # Store the exception for re-raising later read_error = ex logger.warning("unexpected failure to read next chunk", exc_info=True) if not chunk: @@ -257,7 +256,7 @@ class ResponseSource: if current_size > buffer_size: break if len(chunks) == 0: - if read_error and not data_received: + if read_error: raise OperationalError("Failed to read response data from server") from read_error return if decompress: @@ -267,7 +266,6 @@ class ResponseSource: chunk = chunks.popleft() current_size -= len(chunk) if chunk: - data_received = True yield chunk self.gen = buffered() diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/insert.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/insert.py index bcb361afaac..200063fc940 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/insert.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/insert.py @@ -97,6 +97,7 @@ class InsertContext(BaseQueryContext): self.block_row_count = self._calc_block_size() def _calc_block_size(self) -> int: + assert self._data is not None if self.req_block_size: return self.req_block_size row_size = 0 diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/npquery.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/npquery.py index 1c68ed71566..01388f1fb8a 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/npquery.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/npquery.py @@ -1,6 +1,7 @@ import itertools import logging from collections.abc import Generator, Sequence +from typing import Any from clickhouse_connect.driver import options from clickhouse_connect.driver.common import StreamContext, empty_gen @@ -13,19 +14,19 @@ logger = logging.getLogger(__name__) class NumpyResult(Closable): def __init__( self, - block_gen: Generator[Sequence, None, None] = None, + block_gen: Generator[Sequence, None, None] | None = None, column_names: tuple = (), column_types: tuple = (), d_types: Sequence = (), - source: Closable = None, + source: Closable | None = None, ): self.column_names = column_names self.column_types = column_types self.np_types = d_types self.source = source self.query_id = "" - self.summary = {} - self._block_gen = block_gen or empty_gen() + self.summary: dict[str, Any] = {} + self._block_gen: Generator[Sequence, None, None] | None = block_gen or empty_gen() self._numpy_result = None self._df_result = None diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/parser.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/parser.py index 99de3f56cc6..3afe4904fcd 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/parser.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/parser.py @@ -74,7 +74,7 @@ def parse_callable(expr) -> tuple[str, tuple[str | int, ...], str]: return name, tuple(values), expr[pos:].strip() -def parse_enum(expr) -> tuple[tuple[str], tuple[int]]: +def parse_enum(expr) -> tuple[tuple[str, ...], tuple[int, ...]]: """ Parse a ClickHouse enum definition expression of the form ('key1' = 1, 'key2' = 2) :param expr: ClickHouse enum expression/arguments @@ -84,8 +84,8 @@ def parse_enum(expr) -> tuple[tuple[str], tuple[int]]: values = [] pos = expr.find("(") + 1 in_key = False - key = [] - value = [] + key: list[str] = [] + value: list[str] = [] while True: char = expr[pos] pos += 1 @@ -110,8 +110,8 @@ def parse_enum(expr) -> tuple[tuple[str], tuple[int]]: in_key = True else: value.append(char) - values, keys = zip(*sorted(zip(values, keys))) - return tuple(keys), tuple(values) + sorted_values, sorted_keys = zip(*sorted(zip(values, keys))) + return tuple(sorted_keys), tuple(sorted_values) def parse_columns(expr: str): diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/query.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/query.py index 9338e6ff572..f4a94be4cba 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/query.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/query.py @@ -52,7 +52,7 @@ class QueryContext(BaseQueryContext): def __init__( self, query: str | bytes = "", - parameters: dict[str, Any] | None = None, + parameters: Sequence | dict[str, Any] | None = None, settings: dict[str, Any] | None = None, query_formats: dict[str, str] | None = None, column_formats: dict[str, str | dict[str, str]] | None = None, @@ -117,7 +117,7 @@ class QueryContext(BaseQueryContext): self.parameters = parameters or {} self.use_none = True if use_none is None else use_none self.column_oriented = False if column_oriented is None else column_oriented - self.use_numpy = use_numpy + self.use_numpy = use_numpy if use_numpy is not None else False self.max_str_len = 0 if max_str_len is None else max_str_len self.server_tz = server_tz self.apply_server_tz = apply_server_tz @@ -132,7 +132,7 @@ class QueryContext(BaseQueryContext): raise ProgrammingError(f"query_tz {query_tz} is not recognized; {tzutil.TZDATA_HINT}") from ex self.query_tz = query_tz if column_tzs is not None: - resolved_column_tzs = {} + resolved_column_tzs: dict[str, str | tzinfo] = {} for col_name, col_tz in column_tzs.items(): if isinstance(col_tz, str): try: @@ -143,8 +143,8 @@ class QueryContext(BaseQueryContext): resolved_column_tzs[col_name] = col_tz column_tzs = resolved_column_tzs self.column_tzs = column_tzs - self.column_tz = None - self.response_tz = None + self.column_tz: str | tzinfo | None = None + self.response_tz: tzinfo | None = None self.block_info = False self.as_pandas = as_pandas self.streaming = streaming @@ -177,12 +177,12 @@ class QueryContext(BaseQueryContext): def is_command(self) -> bool: return command_re.search(self.uncommented_query) is not None or bare_row_policy_show_re.search(self.uncommented_query) is not None - def set_parameters(self, parameters: dict[str, Any]): + def set_parameters(self, parameters: Sequence | dict[str, Any]): self.parameters = parameters self._update_query() def set_parameter(self, key: str, value: Any): - if not self.parameters: + if not isinstance(self.parameters, dict): self.parameters = {} self.parameters[key] = value self._update_query() @@ -219,7 +219,7 @@ class QueryContext(BaseQueryContext): def updated_copy( self, query: str | bytes | None = None, - parameters: dict[str, Any] | None = None, + parameters: Sequence | dict[str, Any] | None = None, settings: dict[str, Any] | None = None, query_formats: dict[str, str] | None = None, column_formats: dict[str, str | dict[str, str]] | None = None, @@ -245,7 +245,11 @@ class QueryContext(BaseQueryContext): resolved_tz_mode = tz_mode if tz_mode is not None else self.tz_mode return QueryContext( query=query or self.query, - parameters=dict_copy(self.parameters, parameters), + parameters=( + dict_copy(self.parameters, parameters if isinstance(parameters, dict) else None) + if isinstance(self.parameters, dict) + else (parameters if parameters is not None else self.parameters) + ), settings=dict_copy(self.settings, settings), query_formats=dict_copy(self.query_formats, query_formats), column_formats=dict_copy(self.column_formats, column_formats), @@ -283,18 +287,18 @@ class QueryResult(Closable): def __init__( self, - result_set: Matrix = None, - block_gen: Generator[Matrix, None, None] = None, + result_set: Matrix | None = None, + block_gen: Generator[Matrix, None, None] | None = None, column_names: tuple[str, ...] = (), column_types: tuple["ClickHouseType", ...] = (), column_oriented: bool = False, - source: Closable = None, - query_id: str = None, - summary: dict[str, Any] = None, + source: Closable | None = None, + query_id: str | None = None, + summary: dict[str, Any] | None = None, ): - self._result_rows = result_set - self._result_columns = None - self._block_gen = block_gen or empty_gen() + self._result_rows: Matrix | None = result_set + self._result_columns: Matrix | None = None + self._block_gen: Generator[Matrix, None, None] | None = block_gen or empty_gen() self._in_context = False self._query_id = query_id self.column_names = column_names @@ -320,7 +324,7 @@ class QueryResult(Closable): else: self._result_columns = [[] for _ in range(len(self.column_names))] else: - result = [[] for _ in range(len(self.column_names))] + result: list[list[Any]] = [[] for _ in range(len(self.column_names))] with self.column_block_stream as stream: for block in stream: for base, added in zip(result, block): @@ -343,7 +347,7 @@ class QueryResult(Closable): query_id = self.summary.get("query_id") if query_id: return query_id - return self._query_id + return self._query_id or "" def _column_block_stream(self): if self._block_gen is None: @@ -383,13 +387,17 @@ class QueryResult(Closable): return len(self.result_set) @property - def first_item(self) -> dict[str, Any]: + def first_item(self) -> dict[str, Any] | None: + if self.row_count == 0: + return None if self.column_oriented: return {name: col[0] for name, col in zip(self.column_names, self.result_set)} return dict(zip(self.column_names, self.result_set[0])) @property - def first_row(self) -> Sequence[Any]: + def first_row(self) -> Sequence[Any] | None: + if self.row_count == 0: + return None if self.column_oriented: return [col[0] for col in self.result_set] return self.result_set[0] diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/streaming.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/streaming.py index 0a20bfb7697..ddd50f21b3e 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/streaming.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/streaming.py @@ -33,15 +33,15 @@ class StreamingResponseSource(Closable): self.exception_tag = exception_tag # maxsize=10 means max ~10 socket reads buffered - self.queue = AsyncSyncQueue(maxsize=10) + self.queue: AsyncSyncQueue[bytes | Exception] = AsyncSyncQueue(maxsize=10) self._decompressor = None self._decompressor_initialized = False # Multiple accesses to .gen must return the same generator, not create new ones - self._gen_cache = None + self._gen_cache: Iterator[bytes] | None = None - self._producer_task = None + self._producer_task: asyncio.Task | None = None self._producer_started = threading.Event() self._producer_error: Exception | None = None self._producer_completed = False @@ -257,7 +257,7 @@ class StreamingInsertSource: self.transform = transform self.context = context self.loop = loop - self.queue = AsyncSyncQueue(maxsize=maxsize) + self.queue: AsyncSyncQueue[bytes | bytearray | Exception] = AsyncSyncQueue(maxsize=maxsize) self._producer_future = None self._started = False diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/summary.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/summary.py index 8938bb82ed1..6627bc65ad0 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/summary.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/summary.py @@ -3,9 +3,9 @@ from clickhouse_connect.driver.query import QueryResult class QuerySummary: - summary = {} + summary: dict[str, str] = {} - def __init__(self, summary: dict | None = None): + def __init__(self, summary: dict[str, str] | None = None): if summary is not None: self.summary = summary @@ -20,7 +20,7 @@ class QuerySummary: return self.summary.get("query_id", "") def as_query_result(self) -> QueryResult: - data = [] + data: list[int | str] = [] column_names = [] column_types = [] str_type = get_from_name("String") diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/transform.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/transform.py index 5970277b5ee..4a4378d239e 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/transform.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/transform.py @@ -3,7 +3,7 @@ import logging from clickhouse_connect.datatypes import registry from clickhouse_connect.driver.common import write_leb128 from clickhouse_connect.driver.compression import get_compressor -from clickhouse_connect.driver.exceptions import StreamCompleteException, StreamFailureError +from clickhouse_connect.driver.exceptions import OperationalError, StreamCompleteException, StreamFailureError from clickhouse_connect.driver.insert import InsertContext from clickhouse_connect.driver.npquery import NumpyResult from clickhouse_connect.driver.query import QueryContext, QueryResult @@ -71,8 +71,10 @@ class NativeTransform: raise StreamFailureError(error_msg) from None raise StreamFailureError("Stream ended unexpectedly (connection closed by server)") from ex - # Handle async streaming errors (ClientPayloadError from aiohttp) - if ex.__class__.__name__ == "ClientPayloadError": + # A read failure partway through the stream: OperationalError from the sync reader, + # ClientPayloadError from aiohttp. ClickHouse may have written the real error into the + # response body before the connection dropped, so prefer that over the transport error. + if isinstance(ex, OperationalError) or ex.__class__.__name__ == "ClientPayloadError": if source.last_message: error_msg = None exception_tag = getattr(source, "exception_tag", None) @@ -106,7 +108,8 @@ class NativeTransform: @staticmethod def build_insert(context: InsertContext): - compressor = get_compressor(context.compression) + compression = context.compression if isinstance(context.compression, str) else None + compressor = get_compressor(compression) def chunk_gen(): for block in context.next_block(): diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/types.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/types.py index f54b5e86e60..7dadbac32bf 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/types.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/types.py @@ -12,7 +12,7 @@ class Closable(ABC): class ByteSource(Closable): - last_message: bytes = None + last_message: bytes | None = None @abstractmethod def read_leb128(self) -> int: @@ -31,7 +31,7 @@ class ByteSource(Closable): pass @abstractmethod - def read_str_col(self, num_rows: int, encoding: str, nullable: bool = False, null_obj: Any = None): + def read_str_col(self, num_rows: int, encoding: str | None, nullable: bool = False, null_obj: Any = None): pass @abstractmethod diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/tzutil.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/tzutil.py index 7eb330da8bb..f00065df60d 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/tzutil.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/tzutil.py @@ -5,7 +5,7 @@ from datetime import datetime, timedelta, timezone, tzinfo tzlocal = None try: - import tzlocal # Maybe we can use the tzlocal module to get a safe timezone + import tzlocal # type: ignore[no-redef] # Maybe we can use the tzlocal module to get a safe timezone except ImportError: pass @@ -270,7 +270,9 @@ def _detect_local_tz() -> tzinfo: return resolve_zone(env_tz) except zoneinfo.ZoneInfoNotFoundError: pass - return datetime.now().astimezone().tzinfo + local = datetime.now().astimezone().tzinfo + assert local is not None + return local local_tz, local_tz_dst_safe = normalize_timezone(_detect_local_tz()) diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driverc/buffer.pyx b/contrib/python/clickhouse-connect/clickhouse_connect/driverc/buffer.pyx index 1e8bc8e0ca2..9eac5a94d70 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driverc/buffer.pyx +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driverc/buffer.pyx @@ -1,3 +1,4 @@ +# cython: freethreading_compatible = True import sys from typing import Iterable, Any, Optional @@ -22,12 +23,19 @@ cdef char * errors = 'strict' cdef char * utf8 = 'utf8' cdef dict array_templates = {} cdef bint must_swap = sys.byteorder == 'big' -cdef array.array swapper = array.array('Q', [0]) -for c in 'bBuhHiIlLqQfd': +for c in 'bBhHiIlLqQfd': array_templates[c] = array.array(c, []) +cdef inline unsigned long long _bswap_uint64(unsigned long long v): + """Byte-swap a 64-bit unsigned integer for big-endian systems.""" + return (((v & 0xFF) << 56) | (((v >> 8) & 0xFF) << 48) | + (((v >> 16) & 0xFF) << 40) | (((v >> 24) & 0xFF) << 32) | + (((v >> 32) & 0xFF) << 24) | (((v >> 40) & 0xFF) << 16) | + (((v >> 48) & 0xFF) << 8) | ((v >> 56) & 0xFF)) + + cdef class ResponseBuffer: def __init__(self, source): self.slice_sz = 4096 @@ -279,11 +287,11 @@ cdef class ResponseBuffer: @cython.wraparound(False) def read_uint64(self) -> int: cdef ull_wrapper* x + cdef unsigned long long tmp cdef char* b = self.read_bytes_c(8) if must_swap: - memcpy(swapper.data.as_voidptr, b, 8) - swapper.byteswap() - return swapper[0] + memcpy(&tmp, b, 8) + return _bswap_uint64(tmp) x = <ull_wrapper *> b return x.int_value diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driverc/dataconv.pyx b/contrib/python/clickhouse-connect/clickhouse_connect/driverc/dataconv.pyx index 2718ae46454..c82fe85b5e3 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driverc/dataconv.pyx +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driverc/dataconv.pyx @@ -1,3 +1,4 @@ +# cython: freethreading_compatible = True import struct from typing import Sequence, Optional diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driverc/npconv.pyx b/contrib/python/clickhouse-connect/clickhouse_connect/driverc/npconv.pyx index 21568f4f5de..120db6638f5 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driverc/npconv.pyx +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driverc/npconv.pyx @@ -1,3 +1,4 @@ +# cython: freethreading_compatible = True import cython import numpy as np diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/py.typed b/contrib/python/clickhouse-connect/clickhouse_connect/py.typed new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/contrib/python/clickhouse-connect/clickhouse_connect/py.typed diff --git a/contrib/python/clickhouse-connect/ya.make b/contrib/python/clickhouse-connect/ya.make index 34bf63b5b78..800f19f6818 100644 --- a/contrib/python/clickhouse-connect/ya.make +++ b/contrib/python/clickhouse-connect/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(1.3.0) +VERSION(1.4.1) LICENSE(Apache-2.0) @@ -119,6 +119,7 @@ RESOURCE_FILES( .dist-info/METADATA .dist-info/entry_points.txt .dist-info/top_level.txt + clickhouse_connect/py.typed ) END() diff --git a/contrib/python/ipython-genutils/py3/ya.make b/contrib/python/ipython-genutils/py3/ya.make index df370592060..373400ffb8b 100644 --- a/contrib/python/ipython-genutils/py3/ya.make +++ b/contrib/python/ipython-genutils/py3/ya.make @@ -2,6 +2,8 @@ PY3_LIBRARY() +SUBSCRIBER(g:python-contrib) + VERSION(0.2.0) LICENSE(BSD-3-Clause) diff --git a/contrib/python/pickleshare/py3/ya.make b/contrib/python/pickleshare/py3/ya.make index 84629e7786f..cdc377b5045 100644 --- a/contrib/python/pickleshare/py3/ya.make +++ b/contrib/python/pickleshare/py3/ya.make @@ -2,6 +2,8 @@ PY3_LIBRARY() +SUBSCRIBER(g:python-contrib) + VERSION(0.7.5) LICENSE(MIT) diff --git a/contrib/python/simplegeneric/py3/ya.make b/contrib/python/simplegeneric/py3/ya.make index 65e5d02fa45..fe07c8bd96a 100644 --- a/contrib/python/simplegeneric/py3/ya.make +++ b/contrib/python/simplegeneric/py3/ya.make @@ -2,6 +2,8 @@ PY3_LIBRARY() +SUBSCRIBER(g:python-contrib) + VERSION(0.8.1) LICENSE(ZPL-2.1) diff --git a/contrib/python/simplejson/py2/.dist-info/METADATA b/contrib/python/simplejson/py2/.dist-info/METADATA deleted file mode 100644 index e95da1917c6..00000000000 --- a/contrib/python/simplejson/py2/.dist-info/METADATA +++ /dev/null @@ -1,71 +0,0 @@ -Metadata-Version: 2.1 -Name: simplejson -Version: 3.20.2 -Summary: Simple, fast, extensible JSON encoder/decoder for Python -Home-page: https://github.com/simplejson/simplejson -Author: Bob Ippolito -Author-email: [email protected] -License: MIT License -Platform: any -Classifier: Development Status :: 5 - Production/Stable -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: MIT License -Classifier: License :: OSI Approved :: Academic Free License (AFL) -Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 2 -Classifier: Programming Language :: Python :: 2.5 -Classifier: Programming Language :: Python :: 2.6 -Classifier: Programming Language :: Python :: 2.7 -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.3 -Classifier: Programming Language :: Python :: 3.4 -Classifier: Programming Language :: Python :: 3.5 -Classifier: Programming Language :: Python :: 3.6 -Classifier: Programming Language :: Python :: 3.7 -Classifier: Programming Language :: Python :: 3.8 -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Programming Language :: Python :: Implementation :: CPython -Classifier: Programming Language :: Python :: Implementation :: PyPy -Classifier: Topic :: Software Development :: Libraries :: Python Modules -Requires-Python: >=2.5, !=3.0.*, !=3.1.*, !=3.2.* - -simplejson ----------- - -simplejson is a simple, fast, complete, correct and extensible -JSON <http://json.org> encoder and decoder for Python 3.3+ -with legacy support for Python 2.5+. It is pure Python code -with no dependencies, but includes an optional C extension -for a serious speed boost. - -The latest documentation for simplejson can be read online here: -https://simplejson.readthedocs.io/ - -simplejson is the externally maintained development version of the -json library included with Python (since 2.6). This version is tested -with the latest Python 3.8 and maintains backwards compatibility -with Python 3.3+ and the legacy Python 2.5 - Python 2.7 releases. - -The encoder can be specialized to provide serialization in any kind of -situation, without any special support by the objects to be serialized -(somewhat like pickle). This is best done with the ``default`` kwarg -to dumps. - -The decoder can handle incoming JSON strings of any specified encoding -(UTF-8 by default). It can also be specialized to post-process JSON -objects with the ``object_hook`` or ``object_pairs_hook`` kwargs. This -is particularly useful for implementing protocols such as JSON-RPC -that have a richer type system than JSON itself. - -For those of you that have legacy systems to maintain, there is a -very old fork of simplejson in the `python2.2`_ branch that supports -Python 2.2. This is based on a very old version of simplejson, -is not maintained, and should only be used as a last resort. - -.. _python2.2: https://github.com/simplejson/simplejson/tree/python2.2 - - diff --git a/contrib/python/simplejson/py2/.dist-info/top_level.txt b/contrib/python/simplejson/py2/.dist-info/top_level.txt deleted file mode 100644 index 322630ee75f..00000000000 --- a/contrib/python/simplejson/py2/.dist-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -simplejson diff --git a/contrib/python/simplejson/py2/LICENSE.txt b/contrib/python/simplejson/py2/LICENSE.txt deleted file mode 100644 index e05f49c3fd0..00000000000 --- a/contrib/python/simplejson/py2/LICENSE.txt +++ /dev/null @@ -1,79 +0,0 @@ -simplejson is dual-licensed software. It is available under the terms -of the MIT license, or the Academic Free License version 2.1. The full -text of each license agreement is included below. This code is also -licensed to the Python Software Foundation (PSF) under a Contributor -Agreement. - -MIT License -=========== - -Copyright (c) 2006 Bob Ippolito - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -Academic Free License v. 2.1 -============================ - -Copyright (c) 2006 Bob Ippolito. All rights reserved. - -This Academic Free License (the "License") applies to any original work of authorship (the "Original Work") whose owner (the "Licensor") has placed the following notice immediately following the copyright notice for the Original Work: - -Licensed under the Academic Free License version 2.1 - -1) Grant of Copyright License. Licensor hereby grants You a world-wide, royalty-free, non-exclusive, perpetual, sublicenseable license to do the following: - -a) to reproduce the Original Work in copies; - -b) to prepare derivative works ("Derivative Works") based upon the Original Work; - -c) to distribute copies of the Original Work and Derivative Works to the public; - -d) to perform the Original Work publicly; and - -e) to display the Original Work publicly. - -2) Grant of Patent License. Licensor hereby grants You a world-wide, royalty-free, non-exclusive, perpetual, sublicenseable license, under patent claims owned or controlled by the Licensor that are embodied in the Original Work as furnished by the Licensor, to make, use, sell and offer for sale the Original Work and Derivative Works. - -3) Grant of Source Code License. The term "Source Code" means the preferred form of the Original Work for making modifications to it and all available documentation describing how to modify the Original Work. Licensor hereby agrees to provide a machine-readable copy of the Source Code of the Original Work along with each copy of the Original Work that Licensor distributes. Licensor reserves the right to satisfy this obligation by placing a machine-readable copy of the Source Code in an information repository reasonably calculated to permit inexpensive and convenient access by You for as long as Licensor continues to distribute the Original Work, and by publishing the address of that information repository in a notice immediately following the copyright notice that applies to the Original Work. - -4) Exclusions From License Grant. Neither the names of Licensor, nor the names of any contributors to the Original Work, nor any of their trademarks or service marks, may be used to endorse or promote products derived from this Original Work without express prior written permission of the Licensor. Nothing in this License shall be deemed to grant any rights to trademarks, copyrights, patents, trade secrets or any other intellectual property of Licensor except as expressly stated herein. No patent license is granted to make, use, sell or offer to sell embodiments of any patent claims other than the licensed claims defined in Section 2. No right is granted to the trademarks of Licensor even if such marks are included in the Original Work. Nothing in this License shall be interpreted to prohibit Licensor from licensing under different terms from this License any Original Work that Licensor otherwise would have a right to license. - -5) This section intentionally omitted. - -6) Attribution Rights. You must retain, in the Source Code of any Derivative Works that You create, all copyright, patent or trademark notices from the Source Code of the Original Work, as well as any notices of licensing and any descriptive text identified therein as an "Attribution Notice." You must cause the Source Code for any Derivative Works that You create to carry a prominent Attribution Notice reasonably calculated to inform recipients that You have modified the Original Work. - -7) Warranty of Provenance and Disclaimer of Warranty. Licensor warrants that the copyright in and to the Original Work and the patent rights granted herein by Licensor are owned by the Licensor or are sublicensed to You under the terms of this License with the permission of the contributor(s) of those copyrights and patent rights. Except as expressly stated in the immediately proceeding sentence, the Original Work is provided under this License on an "AS IS" BASIS and WITHOUT WARRANTY, either express or implied, including, without limitation, the warranties of NON-INFRINGEMENT, MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY OF THE ORIGINAL WORK IS WITH YOU. This DISCLAIMER OF WARRANTY constitutes an essential part of this License. No license to Original Work is granted hereunder except under this disclaimer. - -8) Limitation of Liability. Under no circumstances and under no legal theory, whether in tort (including negligence), contract, or otherwise, shall the Licensor be liable to any person for any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or the use of the Original Work including, without limitation, damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses. This limitation of liability shall not apply to liability for death or personal injury resulting from Licensor's negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. - -9) Acceptance and Termination. If You distribute copies of the Original Work or a Derivative Work, You must make a reasonable effort under the circumstances to obtain the express assent of recipients to the terms of this License. Nothing else but this License (or another written agreement between Licensor and You) grants You permission to create Derivative Works based upon the Original Work or to exercise any of the rights granted in Section 1 herein, and any attempt to do so except under the terms of this License (or another written agreement between Licensor and You) is expressly prohibited by U.S. copyright law, the equivalent laws of other countries, and by international treaty. Therefore, by exercising any of the rights granted to You in Section 1 herein, You indicate Your acceptance of this License and all of its terms and conditions. - -10) Termination for Patent Action. This License shall terminate automatically and You may no longer exercise any of the rights granted to You by this License as of the date You commence an action, including a cross-claim or counterclaim, against Licensor or any licensee alleging that the Original Work infringes a patent. This termination provision shall not apply for an action alleging patent infringement by combinations of the Original Work with other software or hardware. - -11) Jurisdiction, Venue and Governing Law. Any action or suit relating to this License may be brought only in the courts of a jurisdiction wherein the Licensor resides or in which Licensor conducts its primary business, and under the laws of that jurisdiction excluding its conflict-of-law provisions. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any use of the Original Work outside the scope of this License or after its termination shall be subject to the requirements and penalties of the U.S. Copyright Act, 17 U.S.C. § 101 et seq., the equivalent laws of other countries, and international treaty. This section shall survive the termination of this License. - -12) Attorneys Fees. In any action to enforce the terms of this License or seeking damages relating thereto, the prevailing party shall be entitled to recover its costs and expenses, including, without limitation, reasonable attorneys' fees and costs incurred in connection with such action, including any appeal of such action. This section shall survive the termination of this License. - -13) Miscellaneous. This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. - -14) Definition of "You" in This License. "You" throughout this License, whether in upper or lower case, means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License. For legal entities, "You" includes any entity that controls, is controlled by, or is under common control with you. For purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. - -15) Right to Use. You may use the Original Work in all ways not otherwise restricted or conditioned by this License or by law, and Licensor promises not to interfere with or be responsible for such uses by You. - -This license is Copyright (C) 2003-2004 Lawrence E. Rosen. All rights reserved. Permission is hereby granted to copy and distribute this license without modification. This license may not be modified without the express written permission of its copyright owner. diff --git a/contrib/python/simplejson/py2/README.rst b/contrib/python/simplejson/py2/README.rst deleted file mode 100644 index 6580ddac899..00000000000 --- a/contrib/python/simplejson/py2/README.rst +++ /dev/null @@ -1,34 +0,0 @@ -simplejson ----------- - -simplejson is a simple, fast, complete, correct and extensible -JSON <http://json.org> encoder and decoder for Python 3.3+ -with legacy support for Python 2.5+. It is pure Python code -with no dependencies, but includes an optional C extension -for a serious speed boost. - -The latest documentation for simplejson can be read online here: -https://simplejson.readthedocs.io/ - -simplejson is the externally maintained development version of the -json library included with Python (since 2.6). This version is tested -with the latest Python 3.8 and maintains backwards compatibility -with Python 3.3+ and the legacy Python 2.5 - Python 2.7 releases. - -The encoder can be specialized to provide serialization in any kind of -situation, without any special support by the objects to be serialized -(somewhat like pickle). This is best done with the ``default`` kwarg -to dumps. - -The decoder can handle incoming JSON strings of any specified encoding -(UTF-8 by default). It can also be specialized to post-process JSON -objects with the ``object_hook`` or ``object_pairs_hook`` kwargs. This -is particularly useful for implementing protocols such as JSON-RPC -that have a richer type system than JSON itself. - -For those of you that have legacy systems to maintain, there is a -very old fork of simplejson in the `python2.2`_ branch that supports -Python 2.2. This is based on a very old version of simplejson, -is not maintained, and should only be used as a last resort. - -.. _python2.2: https://github.com/simplejson/simplejson/tree/python2.2 diff --git a/contrib/python/simplejson/py2/simplejson/__init__.py b/contrib/python/simplejson/py2/simplejson/__init__.py deleted file mode 100644 index a526d8c7823..00000000000 --- a/contrib/python/simplejson/py2/simplejson/__init__.py +++ /dev/null @@ -1,562 +0,0 @@ -r"""JSON (JavaScript Object Notation) <http://json.org> is a subset of -JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data -interchange format. - -:mod:`simplejson` exposes an API familiar to users of the standard library -:mod:`marshal` and :mod:`pickle` modules. It is the externally maintained -version of the :mod:`json` library contained in Python 2.6, but maintains -compatibility back to Python 2.5 and (currently) has significant performance -advantages, even without using the optional C extension for speedups. - -Encoding basic Python object hierarchies:: - - >>> import simplejson as json - >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}]) - '["foo", {"bar": ["baz", null, 1.0, 2]}]' - >>> print(json.dumps("\"foo\bar")) - "\"foo\bar" - >>> print(json.dumps(u'\u1234')) - "\u1234" - >>> print(json.dumps('\\')) - "\\" - >>> print(json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True)) - {"a": 0, "b": 0, "c": 0} - >>> from simplejson.compat import StringIO - >>> io = StringIO() - >>> json.dump(['streaming API'], io) - >>> io.getvalue() - '["streaming API"]' - -Compact encoding:: - - >>> import simplejson as json - >>> obj = [1,2,3,{'4': 5, '6': 7}] - >>> json.dumps(obj, separators=(',',':'), sort_keys=True) - '[1,2,3,{"4":5,"6":7}]' - -Pretty printing:: - - >>> import simplejson as json - >>> print(json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=' ')) - { - "4": 5, - "6": 7 - } - -Decoding JSON:: - - >>> import simplejson as json - >>> obj = [u'foo', {u'bar': [u'baz', None, 1.0, 2]}] - >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj - True - >>> json.loads('"\\"foo\\bar"') == u'"foo\x08ar' - True - >>> from simplejson.compat import StringIO - >>> io = StringIO('["streaming API"]') - >>> json.load(io)[0] == 'streaming API' - True - -Specializing JSON object decoding:: - - >>> import simplejson as json - >>> def as_complex(dct): - ... if '__complex__' in dct: - ... return complex(dct['real'], dct['imag']) - ... return dct - ... - >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}', - ... object_hook=as_complex) - (1+2j) - >>> from decimal import Decimal - >>> json.loads('1.1', parse_float=Decimal) == Decimal('1.1') - True - -Specializing JSON object encoding:: - - >>> import simplejson as json - >>> def encode_complex(obj): - ... if isinstance(obj, complex): - ... return [obj.real, obj.imag] - ... raise TypeError('Object of type %s is not JSON serializable' % - ... obj.__class__.__name__) - ... - >>> json.dumps(2 + 1j, default=encode_complex) - '[2.0, 1.0]' - >>> json.JSONEncoder(default=encode_complex).encode(2 + 1j) - '[2.0, 1.0]' - >>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j)) - '[2.0, 1.0]' - -Using simplejson.tool from the shell to validate and pretty-print:: - - $ echo '{"json":"obj"}' | python -m simplejson.tool - { - "json": "obj" - } - $ echo '{ 1.2:3.4}' | python -m simplejson.tool - Expecting property name: line 1 column 3 (char 2) - -Parsing multiple documents serialized as JSON lines (newline-delimited JSON):: - - >>> import simplejson as json - >>> def loads_lines(docs): - ... for doc in docs.splitlines(): - ... yield json.loads(doc) - ... - >>> sum(doc["count"] for doc in loads_lines('{"count":1}\n{"count":2}\n{"count":3}\n')) - 6 - -Serializing multiple objects to JSON lines (newline-delimited JSON):: - - >>> import simplejson as json - >>> def dumps_lines(objs): - ... for obj in objs: - ... yield json.dumps(obj, separators=(',',':')) + '\n' - ... - >>> ''.join(dumps_lines([{'count': 1}, {'count': 2}, {'count': 3}])) - '{"count":1}\n{"count":2}\n{"count":3}\n' - -""" -from __future__ import absolute_import -__version__ = '3.20.2' -__all__ = [ - 'dump', 'dumps', 'load', 'loads', - 'JSONDecoder', 'JSONDecodeError', 'JSONEncoder', - 'OrderedDict', 'simple_first', 'RawJSON' -] - -__author__ = 'Bob Ippolito <[email protected]>' - -from decimal import Decimal - -from .errors import JSONDecodeError -from .raw_json import RawJSON -from .decoder import JSONDecoder -from .encoder import JSONEncoder, JSONEncoderForHTML -def _import_OrderedDict(): - import collections - try: - return collections.OrderedDict - except AttributeError: - from . import ordered_dict - return ordered_dict.OrderedDict -OrderedDict = _import_OrderedDict() - -def _import_c_make_encoder(): - try: - from ._speedups import make_encoder - return make_encoder - except ImportError: - return None - -_default_encoder = JSONEncoder() - -def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, - allow_nan=False, cls=None, indent=None, separators=None, - encoding='utf-8', default=None, use_decimal=True, - namedtuple_as_object=True, tuple_as_array=True, - bigint_as_string=False, sort_keys=False, item_sort_key=None, - for_json=False, ignore_nan=False, int_as_string_bitcount=None, - iterable_as_array=False, **kw): - """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a - ``.write()``-supporting file-like object). - - If *skipkeys* is true then ``dict`` keys that are not basic types - (``str``, ``int``, ``long``, ``float``, ``bool``, ``None``) - will be skipped instead of raising a ``TypeError``. - - If *ensure_ascii* is false (default: ``True``), then the output may - contain non-ASCII characters, so long as they do not need to be escaped - by JSON. When it is true, all non-ASCII characters are escaped. - - If *allow_nan* is true (default: ``False``), then out of range ``float`` - values (``nan``, ``inf``, ``-inf``) will be serialized to - their JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``) - instead of raising a ValueError. See - *ignore_nan* for ECMA-262 compliant behavior. - - If *indent* is a string, then JSON array elements and object members - will be pretty-printed with a newline followed by that string repeated - for each level of nesting. ``None`` (the default) selects the most compact - representation without any newlines. - - If specified, *separators* should be an - ``(item_separator, key_separator)`` tuple. The default is ``(', ', ': ')`` - if *indent* is ``None`` and ``(',', ': ')`` otherwise. To get the most - compact JSON representation, you should specify ``(',', ':')`` to eliminate - whitespace. - - *encoding* is the character encoding for str instances, default is UTF-8. - - *default(obj)* is a function that should return a serializable version - of obj or raise ``TypeError``. The default simply raises ``TypeError``. - - If *use_decimal* is true (default: ``True``) then decimal.Decimal - will be natively serialized to JSON with full precision. - - If *namedtuple_as_object* is true (default: ``True``), - :class:`tuple` subclasses with ``_asdict()`` methods will be encoded - as JSON objects. - - If *tuple_as_array* is true (default: ``True``), - :class:`tuple` (and subclasses) will be encoded as JSON arrays. - - If *iterable_as_array* is true (default: ``False``), - any object not in the above table that implements ``__iter__()`` - will be encoded as a JSON array. - - If *bigint_as_string* is true (default: ``False``), ints 2**53 and higher - or lower than -2**53 will be encoded as strings. This is to avoid the - rounding that happens in Javascript otherwise. Note that this is still a - lossy operation that will not round-trip correctly and should be used - sparingly. - - If *int_as_string_bitcount* is a positive number (n), then int of size - greater than or equal to 2**n or lower than or equal to -2**n will be - encoded as strings. - - If specified, *item_sort_key* is a callable used to sort the items in - each dictionary. This is useful if you want to sort items other than - in alphabetical order by key. This option takes precedence over - *sort_keys*. - - If *sort_keys* is true (default: ``False``), the output of dictionaries - will be sorted by item. - - If *for_json* is true (default: ``False``), objects with a ``for_json()`` - method will use the return value of that method for encoding as JSON - instead of the object. - - If *ignore_nan* is true (default: ``False``), then out of range - :class:`float` values (``nan``, ``inf``, ``-inf``) will be serialized as - ``null`` in compliance with the ECMA-262 specification. If true, this will - override *allow_nan*. - - To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the - ``.default()`` method to serialize additional types), specify it with - the ``cls`` kwarg. NOTE: You should use *default* or *for_json* instead - of subclassing whenever possible. - - """ - # cached encoder - if (not skipkeys and ensure_ascii and - check_circular and not allow_nan and - cls is None and indent is None and separators is None and - encoding == 'utf-8' and default is None and use_decimal - and namedtuple_as_object and tuple_as_array and not iterable_as_array - and not bigint_as_string and not sort_keys - and not item_sort_key and not for_json - and not ignore_nan and int_as_string_bitcount is None - and not kw - ): - iterable = _default_encoder.iterencode(obj) - else: - if cls is None: - cls = JSONEncoder - iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii, - check_circular=check_circular, allow_nan=allow_nan, indent=indent, - separators=separators, encoding=encoding, - default=default, use_decimal=use_decimal, - namedtuple_as_object=namedtuple_as_object, - tuple_as_array=tuple_as_array, - iterable_as_array=iterable_as_array, - bigint_as_string=bigint_as_string, - sort_keys=sort_keys, - item_sort_key=item_sort_key, - for_json=for_json, - ignore_nan=ignore_nan, - int_as_string_bitcount=int_as_string_bitcount, - **kw).iterencode(obj) - # could accelerate with writelines in some versions of Python, at - # a debuggability cost - for chunk in iterable: - fp.write(chunk) - - -def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, - allow_nan=False, cls=None, indent=None, separators=None, - encoding='utf-8', default=None, use_decimal=True, - namedtuple_as_object=True, tuple_as_array=True, - bigint_as_string=False, sort_keys=False, item_sort_key=None, - for_json=False, ignore_nan=False, int_as_string_bitcount=None, - iterable_as_array=False, **kw): - """Serialize ``obj`` to a JSON formatted ``str``. - - If ``skipkeys`` is true then ``dict`` keys that are not basic types - (``str``, ``int``, ``long``, ``float``, ``bool``, ``None``) - will be skipped instead of raising a ``TypeError``. - - If *ensure_ascii* is false (default: ``True``), then the output may - contain non-ASCII characters, so long as they do not need to be escaped - by JSON. When it is true, all non-ASCII characters are escaped. - - If ``check_circular`` is false, then the circular reference check - for container types will be skipped and a circular reference will - result in an ``OverflowError`` (or worse). - - If *allow_nan* is true (default: ``False``), then out of range ``float`` - values (``nan``, ``inf``, ``-inf``) will be serialized to - their JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``) - instead of raising a ValueError. See - *ignore_nan* for ECMA-262 compliant behavior. - - If ``indent`` is a string, then JSON array elements and object members - will be pretty-printed with a newline followed by that string repeated - for each level of nesting. ``None`` (the default) selects the most compact - representation without any newlines. For backwards compatibility with - versions of simplejson earlier than 2.1.0, an integer is also accepted - and is converted to a string with that many spaces. - - If specified, ``separators`` should be an - ``(item_separator, key_separator)`` tuple. The default is ``(', ', ': ')`` - if *indent* is ``None`` and ``(',', ': ')`` otherwise. To get the most - compact JSON representation, you should specify ``(',', ':')`` to eliminate - whitespace. - - ``encoding`` is the character encoding for bytes instances, default is - UTF-8. - - ``default(obj)`` is a function that should return a serializable version - of obj or raise TypeError. The default simply raises TypeError. - - If *use_decimal* is true (default: ``True``) then decimal.Decimal - will be natively serialized to JSON with full precision. - - If *namedtuple_as_object* is true (default: ``True``), - :class:`tuple` subclasses with ``_asdict()`` methods will be encoded - as JSON objects. - - If *tuple_as_array* is true (default: ``True``), - :class:`tuple` (and subclasses) will be encoded as JSON arrays. - - If *iterable_as_array* is true (default: ``False``), - any object not in the above table that implements ``__iter__()`` - will be encoded as a JSON array. - - If *bigint_as_string* is true (not the default), ints 2**53 and higher - or lower than -2**53 will be encoded as strings. This is to avoid the - rounding that happens in Javascript otherwise. - - If *int_as_string_bitcount* is a positive number (n), then int of size - greater than or equal to 2**n or lower than or equal to -2**n will be - encoded as strings. - - If specified, *item_sort_key* is a callable used to sort the items in - each dictionary. This is useful if you want to sort items other than - in alphabetical order by key. This option takes precedence over - *sort_keys*. - - If *sort_keys* is true (default: ``False``), the output of dictionaries - will be sorted by item. - - If *for_json* is true (default: ``False``), objects with a ``for_json()`` - method will use the return value of that method for encoding as JSON - instead of the object. - - If *ignore_nan* is true (default: ``False``), then out of range - :class:`float` values (``nan``, ``inf``, ``-inf``) will be serialized as - ``null`` in compliance with the ECMA-262 specification. If true, this will - override *allow_nan*. - - To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the - ``.default()`` method to serialize additional types), specify it with - the ``cls`` kwarg. NOTE: You should use *default* instead of subclassing - whenever possible. - - """ - # cached encoder - if (not skipkeys and ensure_ascii and - check_circular and not allow_nan and - cls is None and indent is None and separators is None and - encoding == 'utf-8' and default is None and use_decimal - and namedtuple_as_object and tuple_as_array and not iterable_as_array - and not bigint_as_string and not sort_keys - and not item_sort_key and not for_json - and not ignore_nan and int_as_string_bitcount is None - and not kw - ): - return _default_encoder.encode(obj) - if cls is None: - cls = JSONEncoder - return cls( - skipkeys=skipkeys, ensure_ascii=ensure_ascii, - check_circular=check_circular, allow_nan=allow_nan, indent=indent, - separators=separators, encoding=encoding, default=default, - use_decimal=use_decimal, - namedtuple_as_object=namedtuple_as_object, - tuple_as_array=tuple_as_array, - iterable_as_array=iterable_as_array, - bigint_as_string=bigint_as_string, - sort_keys=sort_keys, - item_sort_key=item_sort_key, - for_json=for_json, - ignore_nan=ignore_nan, - int_as_string_bitcount=int_as_string_bitcount, - **kw).encode(obj) - - -_default_decoder = JSONDecoder() - - -def load(fp, encoding=None, cls=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, object_pairs_hook=None, - use_decimal=False, allow_nan=False, **kw): - """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing - a JSON document as `str` or `bytes`) to a Python object. - - *encoding* determines the encoding used to interpret any - `bytes` objects decoded by this instance (``'utf-8'`` by - default). It has no effect when decoding `str` objects. - - *object_hook*, if specified, will be called with the result of every - JSON object decoded and its return value will be used in place of the - given :class:`dict`. This can be used to provide custom - deserializations (e.g. to support JSON-RPC class hinting). - - *object_pairs_hook* is an optional function that will be called with - the result of any object literal decode with an ordered list of pairs. - The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders - that rely on the order that the key and value pairs are decoded (for - example, :func:`collections.OrderedDict` will remember the order of - insertion). If *object_hook* is also defined, the *object_pairs_hook* - takes priority. - - *parse_float*, if specified, will be called with the string of every - JSON float to be decoded. By default, this is equivalent to - ``float(num_str)``. This can be used to use another datatype or parser - for JSON floats (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every - JSON int to be decoded. By default, this is equivalent to - ``int(num_str)``. This can be used to use another datatype or parser - for JSON integers (e.g. :class:`float`). - - *allow_nan*, if True (default false), will allow the parser to - accept the non-standard floats ``NaN``, ``Infinity``, and ``-Infinity`` - and enable the use of the deprecated *parse_constant*. - - If *use_decimal* is true (default: ``False``) then it implies - parse_float=decimal.Decimal for parity with ``dump``. - - *parse_constant*, if specified, will be - called with one of the following strings: ``'-Infinity'``, - ``'Infinity'``, ``'NaN'``. It is not recommended to use this feature, - as it is rare to parse non-compliant JSON containing these values. - - To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` - kwarg. NOTE: You should use *object_hook* or *object_pairs_hook* instead - of subclassing whenever possible. - - """ - return loads(fp.read(), - encoding=encoding, cls=cls, object_hook=object_hook, - parse_float=parse_float, parse_int=parse_int, - parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, - use_decimal=use_decimal, allow_nan=allow_nan, **kw) - - -def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, object_pairs_hook=None, - use_decimal=False, allow_nan=False, **kw): - """Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON - document) to a Python object. - - *encoding* determines the encoding used to interpret any - :class:`bytes` objects decoded by this instance (``'utf-8'`` by - default). It has no effect when decoding :class:`unicode` objects. - - *object_hook*, if specified, will be called with the result of every - JSON object decoded and its return value will be used in place of the - given :class:`dict`. This can be used to provide custom - deserializations (e.g. to support JSON-RPC class hinting). - - *object_pairs_hook* is an optional function that will be called with - the result of any object literal decode with an ordered list of pairs. - The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders - that rely on the order that the key and value pairs are decoded (for - example, :func:`collections.OrderedDict` will remember the order of - insertion). If *object_hook* is also defined, the *object_pairs_hook* - takes priority. - - *parse_float*, if specified, will be called with the string of every - JSON float to be decoded. By default, this is equivalent to - ``float(num_str)``. This can be used to use another datatype or parser - for JSON floats (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every - JSON int to be decoded. By default, this is equivalent to - ``int(num_str)``. This can be used to use another datatype or parser - for JSON integers (e.g. :class:`float`). - - *allow_nan*, if True (default false), will allow the parser to - accept the non-standard floats ``NaN``, ``Infinity``, and ``-Infinity`` - and enable the use of the deprecated *parse_constant*. - - If *use_decimal* is true (default: ``False``) then it implies - parse_float=decimal.Decimal for parity with ``dump``. - - *parse_constant*, if specified, will be - called with one of the following strings: ``'-Infinity'``, - ``'Infinity'``, ``'NaN'``. It is not recommended to use this feature, - as it is rare to parse non-compliant JSON containing these values. - - To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` - kwarg. NOTE: You should use *object_hook* or *object_pairs_hook* instead - of subclassing whenever possible. - - """ - if (cls is None and encoding is None and object_hook is None and - parse_int is None and parse_float is None and - parse_constant is None and object_pairs_hook is None - and not use_decimal and not allow_nan and not kw): - return _default_decoder.decode(s) - if cls is None: - cls = JSONDecoder - if object_hook is not None: - kw['object_hook'] = object_hook - if object_pairs_hook is not None: - kw['object_pairs_hook'] = object_pairs_hook - if parse_float is not None: - kw['parse_float'] = parse_float - if parse_int is not None: - kw['parse_int'] = parse_int - if parse_constant is not None: - kw['parse_constant'] = parse_constant - if use_decimal: - if parse_float is not None: - raise TypeError("use_decimal=True implies parse_float=Decimal") - kw['parse_float'] = Decimal - if allow_nan: - kw['allow_nan'] = True - return cls(encoding=encoding, **kw).decode(s) - - -def _toggle_speedups(enabled): - from . import decoder as dec - from . import encoder as enc - from . import scanner as scan - c_make_encoder = _import_c_make_encoder() - if enabled: - dec.scanstring = dec.c_scanstring or dec.py_scanstring - enc.c_make_encoder = c_make_encoder - enc.encode_basestring_ascii = (enc.c_encode_basestring_ascii or - enc.py_encode_basestring_ascii) - scan.make_scanner = scan.c_make_scanner or scan.py_make_scanner - else: - dec.scanstring = dec.py_scanstring - enc.c_make_encoder = None - enc.encode_basestring_ascii = enc.py_encode_basestring_ascii - scan.make_scanner = scan.py_make_scanner - dec.make_scanner = scan.make_scanner - global _default_decoder - _default_decoder = JSONDecoder() - global _default_encoder - _default_encoder = JSONEncoder() - -def simple_first(kv): - """Helper function to pass to item_sort_key to sort simple - elements to the top, then container elements. - """ - return (isinstance(kv[1], (list, dict, tuple)), kv[0]) diff --git a/contrib/python/simplejson/py2/simplejson/_speedups.c b/contrib/python/simplejson/py2/simplejson/_speedups.c deleted file mode 100644 index 4133b9bc761..00000000000 --- a/contrib/python/simplejson/py2/simplejson/_speedups.c +++ /dev/null @@ -1,3414 +0,0 @@ -/* -*- mode: C; c-file-style: "python"; c-basic-offset: 4 -*- */ -#include "Python.h" -#include "structmember.h" - -#if PY_MAJOR_VERSION >= 3 -#define PyInt_FromSsize_t PyLong_FromSsize_t -#define PyInt_AsSsize_t PyLong_AsSsize_t -#define PyInt_Check(obj) 0 -#define PyInt_CheckExact(obj) 0 -#define JSON_UNICHR Py_UCS4 -#define JSON_InternFromString PyUnicode_InternFromString -#define PyString_GET_SIZE PyUnicode_GET_LENGTH -#define JSON_StringCheck PyUnicode_Check -#define PY2_UNUSED -#define PY3_UNUSED UNUSED -#else /* PY_MAJOR_VERSION >= 3 */ -#define PY2_UNUSED UNUSED -#define PY3_UNUSED -#define JSON_StringCheck(obj) (PyString_Check(obj) || PyUnicode_Check(obj)) -#define PyBytes_Check PyString_Check -#define PyUnicode_READY(obj) 0 -#define PyUnicode_KIND(obj) (sizeof(Py_UNICODE)) -#define PyUnicode_DATA(obj) ((void *)(PyUnicode_AS_UNICODE(obj))) -#define PyUnicode_READ(kind, data, index) ((JSON_UNICHR)((const Py_UNICODE *)(data))[(index)]) -#define PyUnicode_GET_LENGTH PyUnicode_GET_SIZE -#define JSON_UNICHR Py_UNICODE -#define JSON_InternFromString PyString_InternFromString -#endif /* PY_MAJOR_VERSION < 3 */ - -#if PY_VERSION_HEX < 0x03090000 -#if !defined(PyObject_CallNoArgs) -#define PyObject_CallNoArgs(callable) PyObject_CallFunctionObjArgs(callable, NULL); -#endif -#if !defined(PyObject_CallOneArg) -#define PyObject_CallOneArg(callable, arg) PyObject_CallFunctionObjArgs(callable, arg, NULL); -#endif -#endif /* PY_VERSION_HEX < 0x03090000 */ - -#if PY_VERSION_HEX < 0x02070000 -#if !defined(PyOS_string_to_double) -#define PyOS_string_to_double json_PyOS_string_to_double -static double -json_PyOS_string_to_double(const char *s, char **endptr, PyObject *overflow_exception); -static double -json_PyOS_string_to_double(const char *s, char **endptr, PyObject *overflow_exception) -{ - double x; - assert(endptr == NULL); - assert(overflow_exception == NULL); - PyFPE_START_PROTECT("json_PyOS_string_to_double", return -1.0;) - x = PyOS_ascii_atof(s); - PyFPE_END_PROTECT(x) - return x; -} -#endif -#endif /* PY_VERSION_HEX < 0x02070000 */ - -#if PY_VERSION_HEX < 0x02060000 -#if !defined(Py_TYPE) -#define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) -#endif -#if !defined(Py_SIZE) -#define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) -#endif -#if !defined(PyVarObject_HEAD_INIT) -#define PyVarObject_HEAD_INIT(type, size) PyObject_HEAD_INIT(type) size, -#endif -#endif /* PY_VERSION_HEX < 0x02060000 */ - -#ifdef __GNUC__ -#define UNUSED __attribute__((__unused__)) -#else -#define UNUSED -#endif - -#define DEFAULT_ENCODING "utf-8" - -#define PyScanner_Check(op) PyObject_TypeCheck(op, &PyScannerType) -#define PyScanner_CheckExact(op) (Py_TYPE(op) == &PyScannerType) -#define PyEncoder_Check(op) PyObject_TypeCheck(op, &PyEncoderType) -#define PyEncoder_CheckExact(op) (Py_TYPE(op) == &PyEncoderType) - -#define JSON_ALLOW_NAN 1 -#define JSON_IGNORE_NAN 2 - -static PyObject *JSON_Infinity = NULL; -static PyObject *JSON_NegInfinity = NULL; -static PyObject *JSON_NaN = NULL; -static PyObject *JSON_EmptyUnicode = NULL; -#if PY_MAJOR_VERSION < 3 -static PyObject *JSON_EmptyStr = NULL; -#endif - -static PyTypeObject PyScannerType; -static PyTypeObject PyEncoderType; - -typedef struct { - PyObject *large_strings; /* A list of previously accumulated large strings */ - PyObject *small_strings; /* Pending small strings */ -} JSON_Accu; - -static int -JSON_Accu_Init(JSON_Accu *acc); -static int -JSON_Accu_Accumulate(JSON_Accu *acc, PyObject *unicode); -static PyObject * -JSON_Accu_FinishAsList(JSON_Accu *acc); -static void -JSON_Accu_Destroy(JSON_Accu *acc); - -#define ERR_EXPECTING_VALUE "Expecting value" -#define ERR_ARRAY_DELIMITER "Expecting ',' delimiter or ']'" -#define ERR_ARRAY_VALUE_FIRST "Expecting value or ']'" -#define ERR_OBJECT_DELIMITER "Expecting ',' delimiter or '}'" -#define ERR_OBJECT_PROPERTY "Expecting property name enclosed in double quotes" -#define ERR_OBJECT_PROPERTY_FIRST "Expecting property name enclosed in double quotes or '}'" -#define ERR_OBJECT_PROPERTY_DELIMITER "Expecting ':' delimiter" -#define ERR_STRING_UNTERMINATED "Unterminated string starting at" -#define ERR_STRING_CONTROL "Invalid control character %r at" -#define ERR_STRING_ESC1 "Invalid \\X escape sequence %r" -#define ERR_STRING_ESC4 "Invalid \\uXXXX escape sequence" -#define FOR_JSON_METHOD_NAME "for_json" -#define ASDICT_METHOD_NAME "_asdict" - - -typedef struct _PyScannerObject { - PyObject_HEAD - PyObject *encoding; - PyObject *strict_bool; - int strict; - PyObject *object_hook; - PyObject *pairs_hook; - PyObject *parse_float; - PyObject *parse_int; - PyObject *parse_constant; - PyObject *memo; -} PyScannerObject; - -static PyMemberDef scanner_members[] = { - {"encoding", T_OBJECT, offsetof(PyScannerObject, encoding), READONLY, "encoding"}, - {"strict", T_OBJECT, offsetof(PyScannerObject, strict_bool), READONLY, "strict"}, - {"object_hook", T_OBJECT, offsetof(PyScannerObject, object_hook), READONLY, "object_hook"}, - {"object_pairs_hook", T_OBJECT, offsetof(PyScannerObject, pairs_hook), READONLY, "object_pairs_hook"}, - {"parse_float", T_OBJECT, offsetof(PyScannerObject, parse_float), READONLY, "parse_float"}, - {"parse_int", T_OBJECT, offsetof(PyScannerObject, parse_int), READONLY, "parse_int"}, - {"parse_constant", T_OBJECT, offsetof(PyScannerObject, parse_constant), READONLY, "parse_constant"}, - {NULL} -}; - -typedef struct _PyEncoderObject { - PyObject_HEAD - PyObject *markers; - PyObject *defaultfn; - PyObject *encoder; - PyObject *indent; - PyObject *key_separator; - PyObject *item_separator; - PyObject *sort_keys; - PyObject *key_memo; - PyObject *encoding; - PyObject *Decimal; - PyObject *skipkeys_bool; - int skipkeys; - int fast_encode; - /* 0, JSON_ALLOW_NAN, JSON_IGNORE_NAN */ - int allow_or_ignore_nan; - int use_decimal; - int namedtuple_as_object; - int tuple_as_array; - int iterable_as_array; - PyObject *max_long_size; - PyObject *min_long_size; - PyObject *item_sort_key; - PyObject *item_sort_kw; - int for_json; -} PyEncoderObject; - -static PyMemberDef encoder_members[] = { - {"markers", T_OBJECT, offsetof(PyEncoderObject, markers), READONLY, "markers"}, - {"default", T_OBJECT, offsetof(PyEncoderObject, defaultfn), READONLY, "default"}, - {"encoder", T_OBJECT, offsetof(PyEncoderObject, encoder), READONLY, "encoder"}, - {"encoding", T_OBJECT, offsetof(PyEncoderObject, encoder), READONLY, "encoding"}, - {"indent", T_OBJECT, offsetof(PyEncoderObject, indent), READONLY, "indent"}, - {"key_separator", T_OBJECT, offsetof(PyEncoderObject, key_separator), READONLY, "key_separator"}, - {"item_separator", T_OBJECT, offsetof(PyEncoderObject, item_separator), READONLY, "item_separator"}, - {"sort_keys", T_OBJECT, offsetof(PyEncoderObject, sort_keys), READONLY, "sort_keys"}, - /* Python 2.5 does not support T_BOOl */ - {"skipkeys", T_OBJECT, offsetof(PyEncoderObject, skipkeys_bool), READONLY, "skipkeys"}, - {"key_memo", T_OBJECT, offsetof(PyEncoderObject, key_memo), READONLY, "key_memo"}, - {"item_sort_key", T_OBJECT, offsetof(PyEncoderObject, item_sort_key), READONLY, "item_sort_key"}, - {"max_long_size", T_OBJECT, offsetof(PyEncoderObject, max_long_size), READONLY, "max_long_size"}, - {"min_long_size", T_OBJECT, offsetof(PyEncoderObject, min_long_size), READONLY, "min_long_size"}, - {NULL} -}; - -static PyObject * -join_list_unicode(PyObject *lst); -static PyObject * -JSON_ParseEncoding(PyObject *encoding); -static PyObject * -maybe_quote_bigint(PyEncoderObject* s, PyObject *encoded, PyObject *obj); -static Py_ssize_t -ascii_char_size(JSON_UNICHR c); -static Py_ssize_t -ascii_escape_char(JSON_UNICHR c, char *output, Py_ssize_t chars); -static PyObject * -ascii_escape_unicode(PyObject *pystr); -static PyObject * -ascii_escape_str(PyObject *pystr); -static PyObject * -py_encode_basestring_ascii(PyObject* self UNUSED, PyObject *pystr); -#if PY_MAJOR_VERSION < 3 -static PyObject * -join_list_string(PyObject *lst); -static PyObject * -scan_once_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr); -static PyObject * -scanstring_str(PyObject *pystr, Py_ssize_t end, char *encoding, int strict, Py_ssize_t *next_end_ptr); -static PyObject * -_parse_object_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr); -#endif -static PyObject * -scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next_end_ptr); -static PyObject * -scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr); -static PyObject * -_build_rval_index_tuple(PyObject *rval, Py_ssize_t idx); -static PyObject * -scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds); -static void -scanner_dealloc(PyObject *self); -static int -scanner_clear(PyObject *self); -static PyObject * -encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds); -static void -encoder_dealloc(PyObject *self); -static int -encoder_clear(PyObject *self); -static int -is_raw_json(PyObject *obj); -static PyObject * -encoder_stringify_key(PyEncoderObject *s, PyObject *key); -static int -encoder_listencode_list(PyEncoderObject *s, JSON_Accu *rval, PyObject *seq, Py_ssize_t indent_level); -static int -encoder_listencode_obj(PyEncoderObject *s, JSON_Accu *rval, PyObject *obj, Py_ssize_t indent_level); -static int -encoder_listencode_dict(PyEncoderObject *s, JSON_Accu *rval, PyObject *dct, Py_ssize_t indent_level); -static PyObject * -_encoded_const(PyObject *obj); -static void -raise_errmsg(char *msg, PyObject *s, Py_ssize_t end); -static PyObject * -encoder_encode_string(PyEncoderObject *s, PyObject *obj); -static int -_convertPyInt_AsSsize_t(PyObject *o, Py_ssize_t *size_ptr); -static PyObject * -_convertPyInt_FromSsize_t(Py_ssize_t *size_ptr); -static int -_call_json_method(PyObject *obj, const char *method_name, PyObject **result); -static PyObject * -encoder_encode_float(PyEncoderObject *s, PyObject *obj); -static PyObject * -moduleinit(void); - -#define S_CHAR(c) (c >= ' ' && c <= '~' && c != '\\' && c != '"') -#define IS_WHITESPACE(c) (((c) == ' ') || ((c) == '\t') || ((c) == '\n') || ((c) == '\r')) - -#define MIN_EXPANSION 6 - -static PyObject* RawJSONType = NULL; -static int -is_raw_json(PyObject *obj) -{ - return PyObject_IsInstance(obj, RawJSONType) ? 1 : 0; -} - -static int -JSON_Accu_Init(JSON_Accu *acc) -{ - /* Lazily allocated */ - acc->large_strings = NULL; - acc->small_strings = PyList_New(0); - if (acc->small_strings == NULL) - return -1; - return 0; -} - -static int -flush_accumulator(JSON_Accu *acc) -{ - Py_ssize_t nsmall = PyList_GET_SIZE(acc->small_strings); - if (nsmall) { - int ret; - PyObject *joined; - if (acc->large_strings == NULL) { - acc->large_strings = PyList_New(0); - if (acc->large_strings == NULL) - return -1; - } -#if PY_MAJOR_VERSION >= 3 - joined = join_list_unicode(acc->small_strings); -#else /* PY_MAJOR_VERSION >= 3 */ - joined = join_list_string(acc->small_strings); -#endif /* PY_MAJOR_VERSION < 3 */ - if (joined == NULL) - return -1; - if (PyList_SetSlice(acc->small_strings, 0, nsmall, NULL)) { - Py_DECREF(joined); - return -1; - } - ret = PyList_Append(acc->large_strings, joined); - Py_DECREF(joined); - return ret; - } - return 0; -} - -static int -JSON_Accu_Accumulate(JSON_Accu *acc, PyObject *unicode) -{ - Py_ssize_t nsmall; -#if PY_MAJOR_VERSION >= 3 - assert(PyUnicode_Check(unicode)); -#else /* PY_MAJOR_VERSION >= 3 */ - assert(PyString_Check(unicode) || PyUnicode_Check(unicode)); -#endif /* PY_MAJOR_VERSION < 3 */ - - if (PyList_Append(acc->small_strings, unicode)) - return -1; - nsmall = PyList_GET_SIZE(acc->small_strings); - /* Each item in a list of unicode objects has an overhead (in 64-bit - * builds) of: - * - 8 bytes for the list slot - * - 56 bytes for the header of the unicode object - * that is, 64 bytes. 100000 such objects waste more than 6MB - * compared to a single concatenated string. - */ - if (nsmall < 100000) - return 0; - return flush_accumulator(acc); -} - -static PyObject * -JSON_Accu_FinishAsList(JSON_Accu *acc) -{ - int ret; - PyObject *res; - - ret = flush_accumulator(acc); - Py_CLEAR(acc->small_strings); - if (ret) { - Py_CLEAR(acc->large_strings); - return NULL; - } - res = acc->large_strings; - acc->large_strings = NULL; - if (res == NULL) - return PyList_New(0); - return res; -} - -static void -JSON_Accu_Destroy(JSON_Accu *acc) -{ - Py_CLEAR(acc->small_strings); - Py_CLEAR(acc->large_strings); -} - -static int -IS_DIGIT(JSON_UNICHR c) -{ - return c >= '0' && c <= '9'; -} - -static PyObject * -maybe_quote_bigint(PyEncoderObject* s, PyObject *encoded, PyObject *obj) -{ - if (s->max_long_size != Py_None && s->min_long_size != Py_None) { - if (PyObject_RichCompareBool(obj, s->max_long_size, Py_GE) || - PyObject_RichCompareBool(obj, s->min_long_size, Py_LE)) { -#if PY_MAJOR_VERSION >= 3 - PyObject* quoted = PyUnicode_FromFormat("\"%U\"", encoded); -#else - PyObject* quoted = PyString_FromFormat("\"%s\"", - PyString_AsString(encoded)); -#endif - Py_DECREF(encoded); - encoded = quoted; - } - } - - return encoded; -} - -static int -_call_json_method(PyObject *obj, const char *method_name, PyObject **result) -{ - int rval = 0; - PyObject *method = PyObject_GetAttrString(obj, method_name); - if (method == NULL) { - PyErr_Clear(); - return 0; - } - if (PyCallable_Check(method)) { - PyObject *tmp = PyObject_CallNoArgs(method); - if (tmp == NULL && PyErr_ExceptionMatches(PyExc_TypeError)) { - PyErr_Clear(); - } else { - // This will set result to NULL if a TypeError occurred, - // which must be checked by the caller - *result = tmp; - rval = 1; - } - } - Py_DECREF(method); - return rval; -} - -static int -_convertPyInt_AsSsize_t(PyObject *o, Py_ssize_t *size_ptr) -{ - /* PyObject to Py_ssize_t converter */ - *size_ptr = PyInt_AsSsize_t(o); - if (*size_ptr == -1 && PyErr_Occurred()) - return 0; - return 1; -} - -static PyObject * -_convertPyInt_FromSsize_t(Py_ssize_t *size_ptr) -{ - /* Py_ssize_t to PyObject converter */ - return PyInt_FromSsize_t(*size_ptr); -} - -static Py_ssize_t -ascii_escape_char(JSON_UNICHR c, char *output, Py_ssize_t chars) -{ - /* Escape unicode code point c to ASCII escape sequences - in char *output. output must have at least 12 bytes unused to - accommodate an escaped surrogate pair "\uXXXX\uXXXX" */ - if (S_CHAR(c)) { - output[chars++] = (char)c; - } - else { - output[chars++] = '\\'; - switch (c) { - case '\\': output[chars++] = (char)c; break; - case '"': output[chars++] = (char)c; break; - case '\b': output[chars++] = 'b'; break; - case '\f': output[chars++] = 'f'; break; - case '\n': output[chars++] = 'n'; break; - case '\r': output[chars++] = 'r'; break; - case '\t': output[chars++] = 't'; break; - default: -#if PY_MAJOR_VERSION >= 3 || defined(Py_UNICODE_WIDE) - if (c >= 0x10000) { - /* UTF-16 surrogate pair */ - JSON_UNICHR v = c - 0x10000; - c = 0xd800 | ((v >> 10) & 0x3ff); - output[chars++] = 'u'; - output[chars++] = "0123456789abcdef"[(c >> 12) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 8) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 4) & 0xf]; - output[chars++] = "0123456789abcdef"[(c ) & 0xf]; - c = 0xdc00 | (v & 0x3ff); - output[chars++] = '\\'; - } -#endif - output[chars++] = 'u'; - output[chars++] = "0123456789abcdef"[(c >> 12) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 8) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 4) & 0xf]; - output[chars++] = "0123456789abcdef"[(c ) & 0xf]; - } - } - return chars; -} - -static Py_ssize_t -ascii_char_size(JSON_UNICHR c) -{ - if (S_CHAR(c)) { - return 1; - } - else if (c == '\\' || - c == '"' || - c == '\b' || - c == '\f' || - c == '\n' || - c == '\r' || - c == '\t') { - return 2; - } -#if PY_MAJOR_VERSION >= 3 || defined(Py_UNICODE_WIDE) - else if (c >= 0x10000U) { - return 2 * MIN_EXPANSION; - } -#endif - else { - return MIN_EXPANSION; - } -} - -static PyObject * -ascii_escape_unicode(PyObject *pystr) -{ - /* Take a PyUnicode pystr and return a new ASCII-only escaped PyString */ - Py_ssize_t i; - Py_ssize_t input_chars = PyUnicode_GET_LENGTH(pystr); - Py_ssize_t output_size = 2; - Py_ssize_t chars; - PY2_UNUSED int kind = PyUnicode_KIND(pystr); - void *data = PyUnicode_DATA(pystr); - PyObject *rval; - char *output; - - output_size = 2; - for (i = 0; i < input_chars; i++) { - output_size += ascii_char_size(PyUnicode_READ(kind, data, i)); - } -#if PY_MAJOR_VERSION >= 3 - rval = PyUnicode_New(output_size, 127); - if (rval == NULL) { - return NULL; - } - assert(PyUnicode_KIND(rval) == PyUnicode_1BYTE_KIND); - output = (char *)PyUnicode_DATA(rval); -#else - rval = PyString_FromStringAndSize(NULL, output_size); - if (rval == NULL) { - return NULL; - } - output = PyString_AS_STRING(rval); -#endif - chars = 0; - output[chars++] = '"'; - for (i = 0; i < input_chars; i++) { - chars = ascii_escape_char(PyUnicode_READ(kind, data, i), output, chars); - } - output[chars++] = '"'; - assert(chars == output_size); - return rval; -} - -#if PY_MAJOR_VERSION >= 3 - -static PyObject * -ascii_escape_str(PyObject *pystr) -{ - PyObject *rval; - PyObject *input = PyUnicode_DecodeUTF8(PyBytes_AS_STRING(pystr), PyBytes_GET_SIZE(pystr), NULL); - if (input == NULL) - return NULL; - rval = ascii_escape_unicode(input); - Py_DECREF(input); - return rval; -} - -#else /* PY_MAJOR_VERSION >= 3 */ - -static PyObject * -ascii_escape_str(PyObject *pystr) -{ - /* Take a PyString pystr and return a new ASCII-only escaped PyString */ - Py_ssize_t i; - Py_ssize_t input_chars; - Py_ssize_t output_size; - Py_ssize_t chars; - PyObject *rval; - char *output; - char *input_str; - - input_chars = PyString_GET_SIZE(pystr); - input_str = PyString_AS_STRING(pystr); - output_size = 2; - - /* Fast path for a string that's already ASCII */ - for (i = 0; i < input_chars; i++) { - JSON_UNICHR c = (JSON_UNICHR)input_str[i]; - if (c > 0x7f) { - /* We hit a non-ASCII character, bail to unicode mode */ - PyObject *uni; - uni = PyUnicode_DecodeUTF8(input_str, input_chars, "strict"); - if (uni == NULL) { - return NULL; - } - rval = ascii_escape_unicode(uni); - Py_DECREF(uni); - return rval; - } - output_size += ascii_char_size(c); - } - - rval = PyString_FromStringAndSize(NULL, output_size); - if (rval == NULL) { - return NULL; - } - chars = 0; - output = PyString_AS_STRING(rval); - output[chars++] = '"'; - for (i = 0; i < input_chars; i++) { - chars = ascii_escape_char((JSON_UNICHR)input_str[i], output, chars); - } - output[chars++] = '"'; - assert(chars == output_size); - return rval; -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -encoder_stringify_key(PyEncoderObject *s, PyObject *key) -{ - if (PyUnicode_Check(key)) { - Py_INCREF(key); - return key; - } -#if PY_MAJOR_VERSION >= 3 - else if (PyBytes_Check(key) && s->encoding != NULL) { - const char *encoding = PyUnicode_AsUTF8(s->encoding); - if (encoding == NULL) - return NULL; - return PyUnicode_Decode( - PyBytes_AS_STRING(key), - PyBytes_GET_SIZE(key), - encoding, - NULL); - } -#else /* PY_MAJOR_VERSION >= 3 */ - else if (PyString_Check(key)) { - Py_INCREF(key); - return key; - } -#endif /* PY_MAJOR_VERSION < 3 */ - else if (PyFloat_Check(key)) { - return encoder_encode_float(s, key); - } - else if (key == Py_True || key == Py_False || key == Py_None) { - /* This must come before the PyInt_Check because - True and False are also 1 and 0.*/ - return _encoded_const(key); - } - else if (PyInt_Check(key) || PyLong_Check(key)) { - if (!(PyInt_CheckExact(key) || PyLong_CheckExact(key))) { - /* See #118, do not trust custom str/repr */ - PyObject *res; - PyObject *tmp = PyObject_CallOneArg((PyObject *)&PyLong_Type, key); - if (tmp == NULL) { - return NULL; - } - res = PyObject_Str(tmp); - Py_DECREF(tmp); - return res; - } - else { - return PyObject_Str(key); - } - } - else if (s->use_decimal && PyObject_TypeCheck(key, (PyTypeObject *)s->Decimal)) { - return PyObject_Str(key); - } - if (s->skipkeys) { - Py_INCREF(Py_None); - return Py_None; - } - PyErr_Format(PyExc_TypeError, - "keys must be str, int, float, bool or None, " - "not %.100s", key->ob_type->tp_name); - return NULL; -} - -static PyObject * -encoder_dict_iteritems(PyEncoderObject *s, PyObject *dct) -{ - PyObject *items; - PyObject *iter = NULL; - PyObject *lst = NULL; - PyObject *item = NULL; - PyObject *kstr = NULL; - PyObject *sortfun = NULL; - PyObject *sortres; - static PyObject *sortargs = NULL; - - if (sortargs == NULL) { - sortargs = PyTuple_New(0); - if (sortargs == NULL) - return NULL; - } - - if (PyDict_CheckExact(dct)) - items = PyDict_Items(dct); - else - items = PyMapping_Items(dct); - if (items == NULL) - return NULL; - iter = PyObject_GetIter(items); - Py_DECREF(items); - if (iter == NULL) - return NULL; - if (s->item_sort_kw == Py_None) - return iter; - lst = PyList_New(0); - if (lst == NULL) - goto bail; - while ((item = PyIter_Next(iter))) { - PyObject *key, *value; - if (!PyTuple_Check(item) || Py_SIZE(item) != 2) { - PyErr_SetString(PyExc_ValueError, "items must return 2-tuples"); - goto bail; - } - key = PyTuple_GET_ITEM(item, 0); - if (key == NULL) - goto bail; -#if PY_MAJOR_VERSION < 3 - else if (PyString_Check(key)) { - /* item can be added as-is */ - } -#endif /* PY_MAJOR_VERSION < 3 */ - else if (PyUnicode_Check(key)) { - /* item can be added as-is */ - } - else { - PyObject *tpl; - kstr = encoder_stringify_key(s, key); - if (kstr == NULL) - goto bail; - else if (kstr == Py_None) { - /* skipkeys */ - Py_DECREF(kstr); - continue; - } - value = PyTuple_GET_ITEM(item, 1); - if (value == NULL) - goto bail; - tpl = PyTuple_Pack(2, kstr, value); - if (tpl == NULL) - goto bail; - Py_CLEAR(kstr); - Py_DECREF(item); - item = tpl; - } - if (PyList_Append(lst, item)) - goto bail; - Py_DECREF(item); - } - Py_CLEAR(iter); - if (PyErr_Occurred()) - goto bail; - sortfun = PyObject_GetAttrString(lst, "sort"); - if (sortfun == NULL) - goto bail; - sortres = PyObject_Call(sortfun, sortargs, s->item_sort_kw); - if (!sortres) - goto bail; - Py_DECREF(sortres); - Py_CLEAR(sortfun); - iter = PyObject_GetIter(lst); - Py_CLEAR(lst); - return iter; -bail: - Py_XDECREF(sortfun); - Py_XDECREF(kstr); - Py_XDECREF(item); - Py_XDECREF(lst); - Py_XDECREF(iter); - return NULL; -} - -/* Use JSONDecodeError exception to raise a nice looking ValueError subclass */ -static PyObject *JSONDecodeError = NULL; -static void -raise_errmsg(char *msg, PyObject *s, Py_ssize_t end) -{ - PyObject *exc = PyObject_CallFunction(JSONDecodeError, "(zOO&)", msg, s, _convertPyInt_FromSsize_t, &end); - if (exc) { - PyErr_SetObject(JSONDecodeError, exc); - Py_DECREF(exc); - } -} - -static PyObject * -join_list_unicode(PyObject *lst) -{ - /* return u''.join(lst) */ - return PyUnicode_Join(JSON_EmptyUnicode, lst); -} - -#if PY_MAJOR_VERSION >= 3 -#define join_list_string join_list_unicode -#else /* PY_MAJOR_VERSION >= 3 */ -static PyObject * -join_list_string(PyObject *lst) -{ - /* return ''.join(lst) */ - static PyObject *joinfn = NULL; - if (joinfn == NULL) { - joinfn = PyObject_GetAttrString(JSON_EmptyStr, "join"); - if (joinfn == NULL) - return NULL; - } - return PyObject_CallOneArg(joinfn, lst); -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -_build_rval_index_tuple(PyObject *rval, Py_ssize_t idx) -{ - /* return (rval, idx) tuple, stealing reference to rval */ - PyObject *tpl; - PyObject *pyidx; - /* - steal a reference to rval, returns (rval, idx) - */ - if (rval == NULL) { - assert(PyErr_Occurred()); - return NULL; - } - pyidx = PyInt_FromSsize_t(idx); - if (pyidx == NULL) { - Py_DECREF(rval); - return NULL; - } - tpl = PyTuple_New(2); - if (tpl == NULL) { - Py_DECREF(pyidx); - Py_DECREF(rval); - return NULL; - } - PyTuple_SET_ITEM(tpl, 0, rval); - PyTuple_SET_ITEM(tpl, 1, pyidx); - return tpl; -} - -#define APPEND_OLD_CHUNK \ - if (chunk != NULL) { \ - if (chunks == NULL) { \ - chunks = PyList_New(0); \ - if (chunks == NULL) { \ - goto bail; \ - } \ - } \ - if (PyList_Append(chunks, chunk)) { \ - goto bail; \ - } \ - Py_CLEAR(chunk); \ - } - -#if PY_MAJOR_VERSION < 3 -static PyObject * -scanstring_str(PyObject *pystr, Py_ssize_t end, char *encoding, int strict, Py_ssize_t *next_end_ptr) -{ - /* Read the JSON string from PyString pystr. - end is the index of the first character after the quote. - encoding is the encoding of pystr (must be an ASCII superset) - if strict is zero then literal control characters are allowed - *next_end_ptr is a return-by-reference index of the character - after the end quote - - Return value is a new PyString (if ASCII-only) or PyUnicode - */ - PyObject *rval; - Py_ssize_t len = PyString_GET_SIZE(pystr); - Py_ssize_t begin = end - 1; - Py_ssize_t next = begin; - int has_unicode = 0; - char *buf = PyString_AS_STRING(pystr); - PyObject *chunks = NULL; - PyObject *chunk = NULL; - PyObject *strchunk = NULL; - - if (len == end) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - else if (end < 0 || len < end) { - PyErr_SetString(PyExc_ValueError, "end is out of bounds"); - goto bail; - } - while (1) { - /* Find the end of the string or the next escape */ - Py_UNICODE c = 0; - for (next = end; next < len; next++) { - c = (unsigned char)buf[next]; - if (c == '"' || c == '\\') { - break; - } - else if (strict && c <= 0x1f) { - raise_errmsg(ERR_STRING_CONTROL, pystr, next); - goto bail; - } - else if (c > 0x7f) { - has_unicode = 1; - } - } - if (!(c == '"' || c == '\\')) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - /* Pick up this chunk if it's not zero length */ - if (next != end) { - APPEND_OLD_CHUNK - strchunk = PyString_FromStringAndSize(&buf[end], next - end); - if (strchunk == NULL) { - goto bail; - } - if (has_unicode) { - chunk = PyUnicode_FromEncodedObject(strchunk, encoding, NULL); - Py_DECREF(strchunk); - if (chunk == NULL) { - goto bail; - } - } - else { - chunk = strchunk; - } - } - next++; - if (c == '"') { - end = next; - break; - } - if (next == len) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - c = buf[next]; - if (c != 'u') { - /* Non-unicode backslash escapes */ - end = next + 1; - switch (c) { - case '"': break; - case '\\': break; - case '/': break; - case 'b': c = '\b'; break; - case 'f': c = '\f'; break; - case 'n': c = '\n'; break; - case 'r': c = '\r'; break; - case 't': c = '\t'; break; - default: c = 0; - } - if (c == 0) { - raise_errmsg(ERR_STRING_ESC1, pystr, end - 2); - goto bail; - } - } - else { - c = 0; - next++; - end = next + 4; - if (end >= len) { - raise_errmsg(ERR_STRING_ESC4, pystr, next - 1); - goto bail; - } - /* Decode 4 hex digits */ - for (; next < end; next++) { - JSON_UNICHR digit = (JSON_UNICHR)buf[next]; - c <<= 4; - switch (digit) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - c |= (digit - '0'); break; - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': - c |= (digit - 'a' + 10); break; - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': - c |= (digit - 'A' + 10); break; - default: - raise_errmsg(ERR_STRING_ESC4, pystr, end - 5); - goto bail; - } - } -#if defined(Py_UNICODE_WIDE) - /* Surrogate pair */ - if ((c & 0xfc00) == 0xd800) { - if (end + 6 < len && buf[next] == '\\' && buf[next+1] == 'u') { - JSON_UNICHR c2 = 0; - end += 6; - /* Decode 4 hex digits */ - for (next += 2; next < end; next++) { - c2 <<= 4; - JSON_UNICHR digit = buf[next]; - switch (digit) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - c2 |= (digit - '0'); break; - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': - c2 |= (digit - 'a' + 10); break; - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': - c2 |= (digit - 'A' + 10); break; - default: - raise_errmsg(ERR_STRING_ESC4, pystr, end - 5); - goto bail; - } - } - if ((c2 & 0xfc00) != 0xdc00) { - /* not a low surrogate, rewind */ - end -= 6; - next = end; - } - else { - c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00)); - } - } - } -#endif /* Py_UNICODE_WIDE */ - } - if (c > 0x7f) { - has_unicode = 1; - } - APPEND_OLD_CHUNK - if (has_unicode) { - chunk = PyUnicode_FromOrdinal(c); - if (chunk == NULL) { - goto bail; - } - } - else { - char c_char = Py_CHARMASK(c); - chunk = PyString_FromStringAndSize(&c_char, 1); - if (chunk == NULL) { - goto bail; - } - } - } - - if (chunks == NULL) { - if (chunk != NULL) - rval = chunk; - else { - rval = JSON_EmptyStr; - Py_INCREF(rval); - } - } - else { - APPEND_OLD_CHUNK - rval = join_list_string(chunks); - if (rval == NULL) { - goto bail; - } - Py_CLEAR(chunks); - } - - *next_end_ptr = end; - return rval; -bail: - *next_end_ptr = -1; - Py_XDECREF(chunk); - Py_XDECREF(chunks); - return NULL; -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next_end_ptr) -{ - /* Read the JSON string from PyUnicode pystr. - end is the index of the first character after the quote. - if strict is zero then literal control characters are allowed - *next_end_ptr is a return-by-reference index of the character - after the end quote - - Return value is a new PyUnicode - */ - PyObject *rval; - Py_ssize_t begin = end - 1; - Py_ssize_t next = begin; - PY2_UNUSED int kind = PyUnicode_KIND(pystr); - Py_ssize_t len = PyUnicode_GET_LENGTH(pystr); - void *buf = PyUnicode_DATA(pystr); - PyObject *chunks = NULL; - PyObject *chunk = NULL; - - if (len == end) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - else if (end < 0 || len < end) { - PyErr_SetString(PyExc_ValueError, "end is out of bounds"); - goto bail; - } - while (1) { - /* Find the end of the string or the next escape */ - JSON_UNICHR c = 0; - for (next = end; next < len; next++) { - c = PyUnicode_READ(kind, buf, next); - if (c == '"' || c == '\\') { - break; - } - else if (strict && c <= 0x1f) { - raise_errmsg(ERR_STRING_CONTROL, pystr, next); - goto bail; - } - } - if (!(c == '"' || c == '\\')) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - /* Pick up this chunk if it's not zero length */ - if (next != end) { - APPEND_OLD_CHUNK -#if PY_MAJOR_VERSION < 3 - chunk = PyUnicode_FromUnicode(&((const Py_UNICODE *)buf)[end], next - end); -#else - chunk = PyUnicode_Substring(pystr, end, next); -#endif - if (chunk == NULL) { - goto bail; - } - } - next++; - if (c == '"') { - end = next; - break; - } - if (next == len) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - c = PyUnicode_READ(kind, buf, next); - if (c != 'u') { - /* Non-unicode backslash escapes */ - end = next + 1; - switch (c) { - case '"': break; - case '\\': break; - case '/': break; - case 'b': c = '\b'; break; - case 'f': c = '\f'; break; - case 'n': c = '\n'; break; - case 'r': c = '\r'; break; - case 't': c = '\t'; break; - default: c = 0; - } - if (c == 0) { - raise_errmsg(ERR_STRING_ESC1, pystr, end - 2); - goto bail; - } - } - else { - c = 0; - next++; - end = next + 4; - if (end >= len) { - raise_errmsg(ERR_STRING_ESC4, pystr, next - 1); - goto bail; - } - /* Decode 4 hex digits */ - for (; next < end; next++) { - JSON_UNICHR digit = PyUnicode_READ(kind, buf, next); - c <<= 4; - switch (digit) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - c |= (digit - '0'); break; - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': - c |= (digit - 'a' + 10); break; - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': - c |= (digit - 'A' + 10); break; - default: - raise_errmsg(ERR_STRING_ESC4, pystr, end - 5); - goto bail; - } - } -#if PY_MAJOR_VERSION >= 3 || defined(Py_UNICODE_WIDE) - /* Surrogate pair */ - if ((c & 0xfc00) == 0xd800) { - JSON_UNICHR c2 = 0; - if (end + 6 < len && - PyUnicode_READ(kind, buf, next) == '\\' && - PyUnicode_READ(kind, buf, next + 1) == 'u') { - end += 6; - /* Decode 4 hex digits */ - for (next += 2; next < end; next++) { - JSON_UNICHR digit = PyUnicode_READ(kind, buf, next); - c2 <<= 4; - switch (digit) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - c2 |= (digit - '0'); break; - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': - c2 |= (digit - 'a' + 10); break; - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': - c2 |= (digit - 'A' + 10); break; - default: - raise_errmsg(ERR_STRING_ESC4, pystr, end - 5); - goto bail; - } - } - if ((c2 & 0xfc00) != 0xdc00) { - /* not a low surrogate, rewind */ - end -= 6; - next = end; - } - else { - c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00)); - } - } - } -#endif - } - APPEND_OLD_CHUNK - chunk = PyUnicode_FromOrdinal(c); - if (chunk == NULL) { - goto bail; - } - } - - if (chunks == NULL) { - if (chunk != NULL) - rval = chunk; - else { - rval = JSON_EmptyUnicode; - Py_INCREF(rval); - } - } - else { - APPEND_OLD_CHUNK - rval = join_list_unicode(chunks); - if (rval == NULL) { - goto bail; - } - Py_CLEAR(chunks); - } - *next_end_ptr = end; - return rval; -bail: - *next_end_ptr = -1; - Py_XDECREF(chunk); - Py_XDECREF(chunks); - return NULL; -} - -PyDoc_STRVAR(pydoc_scanstring, - "scanstring(basestring, end, encoding, strict=True) -> (str, end)\n" - "\n" - "Scan the string s for a JSON string. End is the index of the\n" - "character in s after the quote that started the JSON string.\n" - "Unescapes all valid JSON string escape sequences and raises ValueError\n" - "on attempt to decode an invalid string. If strict is False then literal\n" - "control characters are allowed in the string.\n" - "\n" - "Returns a tuple of the decoded string and the index of the character in s\n" - "after the end quote." -); - -static PyObject * -py_scanstring(PyObject* self UNUSED, PyObject *args) -{ - PyObject *pystr; - PyObject *rval; - Py_ssize_t end; - Py_ssize_t next_end = -1; - char *encoding = NULL; - int strict = 1; - if (!PyArg_ParseTuple(args, "OO&|zi:scanstring", &pystr, _convertPyInt_AsSsize_t, &end, &encoding, &strict)) { - return NULL; - } - if (encoding == NULL) { - encoding = DEFAULT_ENCODING; - } - if (PyUnicode_Check(pystr)) { - if (PyUnicode_READY(pystr)) - return NULL; - rval = scanstring_unicode(pystr, end, strict, &next_end); - } -#if PY_MAJOR_VERSION < 3 - /* Using a bytes input is unsupported for scanning in Python 3. - It is coerced to str in the decoder before it gets here. */ - else if (PyString_Check(pystr)) { - rval = scanstring_str(pystr, end, encoding, strict, &next_end); - } -#endif - else { - PyErr_Format(PyExc_TypeError, - "first argument must be a string, not %.80s", - Py_TYPE(pystr)->tp_name); - return NULL; - } - return _build_rval_index_tuple(rval, next_end); -} - -PyDoc_STRVAR(pydoc_encode_basestring_ascii, - "encode_basestring_ascii(basestring) -> str\n" - "\n" - "Return an ASCII-only JSON representation of a Python string" -); - -static PyObject * -py_encode_basestring_ascii(PyObject* self UNUSED, PyObject *pystr) -{ - /* Return an ASCII-only JSON representation of a Python string */ - /* METH_O */ - if (PyBytes_Check(pystr)) { - return ascii_escape_str(pystr); - } - else if (PyUnicode_Check(pystr)) { - if (PyUnicode_READY(pystr)) - return NULL; - return ascii_escape_unicode(pystr); - } - else { - PyErr_Format(PyExc_TypeError, - "first argument must be a string, not %.80s", - Py_TYPE(pystr)->tp_name); - return NULL; - } -} - -static void -scanner_dealloc(PyObject *self) -{ - /* bpo-31095: UnTrack is needed before calling any callbacks */ - PyObject_GC_UnTrack(self); - scanner_clear(self); - Py_TYPE(self)->tp_free(self); -} - -static int -scanner_traverse(PyObject *self, visitproc visit, void *arg) -{ - PyScannerObject *s; - assert(PyScanner_Check(self)); - s = (PyScannerObject *)self; - Py_VISIT(s->encoding); - Py_VISIT(s->strict_bool); - Py_VISIT(s->object_hook); - Py_VISIT(s->pairs_hook); - Py_VISIT(s->parse_float); - Py_VISIT(s->parse_int); - Py_VISIT(s->parse_constant); - Py_VISIT(s->memo); - return 0; -} - -static int -scanner_clear(PyObject *self) -{ - PyScannerObject *s; - assert(PyScanner_Check(self)); - s = (PyScannerObject *)self; - Py_CLEAR(s->encoding); - Py_CLEAR(s->strict_bool); - Py_CLEAR(s->object_hook); - Py_CLEAR(s->pairs_hook); - Py_CLEAR(s->parse_float); - Py_CLEAR(s->parse_int); - Py_CLEAR(s->parse_constant); - Py_CLEAR(s->memo); - return 0; -} - -#if PY_MAJOR_VERSION < 3 -static PyObject * -_parse_object_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON object from PyString pystr. - idx is the index of the first character after the opening curly brace. - *next_idx_ptr is a return-by-reference index to the first character after - the closing curly brace. - - Returns a new PyObject (usually a dict, but object_hook or - object_pairs_hook can change that) - */ - char *str = PyString_AS_STRING(pystr); - Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1; - PyObject *rval = NULL; - PyObject *pairs = NULL; - PyObject *item; - PyObject *key = NULL; - PyObject *val = NULL; - char *encoding = PyString_AS_STRING(s->encoding); - int has_pairs_hook = (s->pairs_hook != Py_None); - int did_parse = 0; - Py_ssize_t next_idx; - if (has_pairs_hook) { - pairs = PyList_New(0); - if (pairs == NULL) - return NULL; - } - else { - rval = PyDict_New(); - if (rval == NULL) - return NULL; - } - - /* skip whitespace after { */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* only loop if the object is non-empty */ - if (idx <= end_idx && str[idx] != '}') { - int trailing_delimiter = 0; - while (idx <= end_idx) { - PyObject *memokey; - trailing_delimiter = 0; - - /* read key */ - if (str[idx] != '"') { - raise_errmsg(ERR_OBJECT_PROPERTY, pystr, idx); - goto bail; - } - key = scanstring_str(pystr, idx + 1, encoding, s->strict, &next_idx); - if (key == NULL) - goto bail; - memokey = PyDict_GetItem(s->memo, key); - if (memokey != NULL) { - Py_INCREF(memokey); - Py_DECREF(key); - key = memokey; - } - else { - if (PyDict_SetItem(s->memo, key, key) < 0) - goto bail; - } - idx = next_idx; - - /* skip whitespace between key and : delimiter, read :, skip whitespace */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - if (idx > end_idx || str[idx] != ':') { - raise_errmsg(ERR_OBJECT_PROPERTY_DELIMITER, pystr, idx); - goto bail; - } - idx++; - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* read any JSON data type */ - val = scan_once_str(s, pystr, idx, &next_idx); - if (val == NULL) - goto bail; - - if (has_pairs_hook) { - item = PyTuple_Pack(2, key, val); - if (item == NULL) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - if (PyList_Append(pairs, item) == -1) { - Py_DECREF(item); - goto bail; - } - Py_DECREF(item); - } - else { - if (PyDict_SetItem(rval, key, val) < 0) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - } - idx = next_idx; - - /* skip whitespace before } or , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* bail if the object is closed or we didn't get the , delimiter */ - did_parse = 1; - if (idx > end_idx) break; - if (str[idx] == '}') { - break; - } - else if (str[idx] != ',') { - raise_errmsg(ERR_OBJECT_DELIMITER, pystr, idx); - goto bail; - } - idx++; - - /* skip whitespace after , delimiter */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - trailing_delimiter = 1; - } - if (trailing_delimiter) { - raise_errmsg(ERR_OBJECT_PROPERTY, pystr, idx); - goto bail; - } - } - /* verify that idx < end_idx, str[idx] should be '}' */ - if (idx > end_idx || str[idx] != '}') { - if (did_parse) { - raise_errmsg(ERR_OBJECT_DELIMITER, pystr, idx); - } else { - raise_errmsg(ERR_OBJECT_PROPERTY_FIRST, pystr, idx); - } - goto bail; - } - - /* if pairs_hook is not None: rval = object_pairs_hook(pairs) */ - if (s->pairs_hook != Py_None) { - val = PyObject_CallOneArg(s->pairs_hook, pairs); - if (val == NULL) - goto bail; - Py_DECREF(pairs); - *next_idx_ptr = idx + 1; - return val; - } - - /* if object_hook is not None: rval = object_hook(rval) */ - if (s->object_hook != Py_None) { - val = PyObject_CallOneArg(s->object_hook, rval); - if (val == NULL) - goto bail; - Py_DECREF(rval); - rval = val; - val = NULL; - } - *next_idx_ptr = idx + 1; - return rval; -bail: - Py_XDECREF(rval); - Py_XDECREF(key); - Py_XDECREF(val); - Py_XDECREF(pairs); - return NULL; -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -_parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON object from PyUnicode pystr. - idx is the index of the first character after the opening curly brace. - *next_idx_ptr is a return-by-reference index to the first character after - the closing curly brace. - - Returns a new PyObject (usually a dict, but object_hook can change that) - */ - void *str = PyUnicode_DATA(pystr); - Py_ssize_t end_idx = PyUnicode_GET_LENGTH(pystr) - 1; - PY2_UNUSED int kind = PyUnicode_KIND(pystr); - PyObject *rval = NULL; - PyObject *pairs = NULL; - PyObject *item; - PyObject *key = NULL; - PyObject *val = NULL; - int has_pairs_hook = (s->pairs_hook != Py_None); - int did_parse = 0; - Py_ssize_t next_idx; - - if (has_pairs_hook) { - pairs = PyList_New(0); - if (pairs == NULL) - return NULL; - } - else { - rval = PyDict_New(); - if (rval == NULL) - return NULL; - } - - /* skip whitespace after { */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - - /* only loop if the object is non-empty */ - if (idx <= end_idx && PyUnicode_READ(kind, str, idx) != '}') { - int trailing_delimiter = 0; - while (idx <= end_idx) { - PyObject *memokey; - trailing_delimiter = 0; - - /* read key */ - if (PyUnicode_READ(kind, str, idx) != '"') { - raise_errmsg(ERR_OBJECT_PROPERTY, pystr, idx); - goto bail; - } - key = scanstring_unicode(pystr, idx + 1, s->strict, &next_idx); - if (key == NULL) - goto bail; - memokey = PyDict_GetItem(s->memo, key); - if (memokey != NULL) { - Py_INCREF(memokey); - Py_DECREF(key); - key = memokey; - } - else { - if (PyDict_SetItem(s->memo, key, key) < 0) - goto bail; - } - idx = next_idx; - - /* skip whitespace between key and : delimiter, read :, skip - whitespace */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - if (idx > end_idx || PyUnicode_READ(kind, str, idx) != ':') { - raise_errmsg(ERR_OBJECT_PROPERTY_DELIMITER, pystr, idx); - goto bail; - } - idx++; - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - - /* read any JSON term */ - val = scan_once_unicode(s, pystr, idx, &next_idx); - if (val == NULL) - goto bail; - - if (has_pairs_hook) { - item = PyTuple_Pack(2, key, val); - if (item == NULL) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - if (PyList_Append(pairs, item) == -1) { - Py_DECREF(item); - goto bail; - } - Py_DECREF(item); - } - else { - if (PyDict_SetItem(rval, key, val) < 0) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - } - idx = next_idx; - - /* skip whitespace before } or , */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - - /* bail if the object is closed or we didn't get the , - delimiter */ - did_parse = 1; - if (idx > end_idx) break; - if (PyUnicode_READ(kind, str, idx) == '}') { - break; - } - else if (PyUnicode_READ(kind, str, idx) != ',') { - raise_errmsg(ERR_OBJECT_DELIMITER, pystr, idx); - goto bail; - } - idx++; - - /* skip whitespace after , delimiter */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - trailing_delimiter = 1; - } - if (trailing_delimiter) { - raise_errmsg(ERR_OBJECT_PROPERTY, pystr, idx); - goto bail; - } - } - - /* verify that idx < end_idx, str[idx] should be '}' */ - if (idx > end_idx || PyUnicode_READ(kind, str, idx) != '}') { - if (did_parse) { - raise_errmsg(ERR_OBJECT_DELIMITER, pystr, idx); - } else { - raise_errmsg(ERR_OBJECT_PROPERTY_FIRST, pystr, idx); - } - goto bail; - } - - /* if pairs_hook is not None: rval = object_pairs_hook(pairs) */ - if (s->pairs_hook != Py_None) { - val = PyObject_CallOneArg(s->pairs_hook, pairs); - if (val == NULL) - goto bail; - Py_DECREF(pairs); - *next_idx_ptr = idx + 1; - return val; - } - - /* if object_hook is not None: rval = object_hook(rval) */ - if (s->object_hook != Py_None) { - val = PyObject_CallOneArg(s->object_hook, rval); - if (val == NULL) - goto bail; - Py_DECREF(rval); - rval = val; - val = NULL; - } - *next_idx_ptr = idx + 1; - return rval; -bail: - Py_XDECREF(rval); - Py_XDECREF(key); - Py_XDECREF(val); - Py_XDECREF(pairs); - return NULL; -} - -#if PY_MAJOR_VERSION < 3 -static PyObject * -_parse_array_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON array from PyString pystr. - idx is the index of the first character after the opening brace. - *next_idx_ptr is a return-by-reference index to the first character after - the closing brace. - - Returns a new PyList - */ - char *str = PyString_AS_STRING(pystr); - Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1; - PyObject *val = NULL; - PyObject *rval = PyList_New(0); - Py_ssize_t next_idx; - if (rval == NULL) - return NULL; - - /* skip whitespace after [ */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* only loop if the array is non-empty */ - if (idx <= end_idx && str[idx] != ']') { - int trailing_delimiter = 0; - while (idx <= end_idx) { - trailing_delimiter = 0; - /* read any JSON term and de-tuplefy the (rval, idx) */ - val = scan_once_str(s, pystr, idx, &next_idx); - if (val == NULL) { - goto bail; - } - - if (PyList_Append(rval, val) == -1) - goto bail; - - Py_CLEAR(val); - idx = next_idx; - - /* skip whitespace between term and , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* bail if the array is closed or we didn't get the , delimiter */ - if (idx > end_idx) break; - if (str[idx] == ']') { - break; - } - else if (str[idx] != ',') { - raise_errmsg(ERR_ARRAY_DELIMITER, pystr, idx); - goto bail; - } - idx++; - - /* skip whitespace after , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - trailing_delimiter = 1; - } - if (trailing_delimiter) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx); - goto bail; - } - } - - /* verify that idx < end_idx, str[idx] should be ']' */ - if (idx > end_idx || str[idx] != ']') { - if (PyList_GET_SIZE(rval)) { - raise_errmsg(ERR_ARRAY_DELIMITER, pystr, idx); - } else { - raise_errmsg(ERR_ARRAY_VALUE_FIRST, pystr, idx); - } - goto bail; - } - *next_idx_ptr = idx + 1; - return rval; -bail: - Py_XDECREF(val); - Py_DECREF(rval); - return NULL; -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -_parse_array_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON array from PyString pystr. - idx is the index of the first character after the opening brace. - *next_idx_ptr is a return-by-reference index to the first character after - the closing brace. - - Returns a new PyList - */ - PY2_UNUSED int kind = PyUnicode_KIND(pystr); - void *str = PyUnicode_DATA(pystr); - Py_ssize_t end_idx = PyUnicode_GET_LENGTH(pystr) - 1; - PyObject *val = NULL; - PyObject *rval = PyList_New(0); - Py_ssize_t next_idx; - if (rval == NULL) - return NULL; - - /* skip whitespace after [ */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - - /* only loop if the array is non-empty */ - if (idx <= end_idx && PyUnicode_READ(kind, str, idx) != ']') { - int trailing_delimiter = 0; - while (idx <= end_idx) { - trailing_delimiter = 0; - /* read any JSON term */ - val = scan_once_unicode(s, pystr, idx, &next_idx); - if (val == NULL) { - goto bail; - } - - if (PyList_Append(rval, val) == -1) - goto bail; - - Py_CLEAR(val); - idx = next_idx; - - /* skip whitespace between term and , */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - - /* bail if the array is closed or we didn't get the , delimiter */ - if (idx > end_idx) break; - if (PyUnicode_READ(kind, str, idx) == ']') { - break; - } - else if (PyUnicode_READ(kind, str, idx) != ',') { - raise_errmsg(ERR_ARRAY_DELIMITER, pystr, idx); - goto bail; - } - idx++; - - /* skip whitespace after , */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - trailing_delimiter = 1; - } - if (trailing_delimiter) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx); - goto bail; - } - } - - /* verify that idx < end_idx, str[idx] should be ']' */ - if (idx > end_idx || PyUnicode_READ(kind, str, idx) != ']') { - if (PyList_GET_SIZE(rval)) { - raise_errmsg(ERR_ARRAY_DELIMITER, pystr, idx); - } else { - raise_errmsg(ERR_ARRAY_VALUE_FIRST, pystr, idx); - } - goto bail; - } - *next_idx_ptr = idx + 1; - return rval; -bail: - Py_XDECREF(val); - Py_DECREF(rval); - return NULL; -} - -static PyObject * -_parse_constant(PyScannerObject *s, PyObject *pystr, PyObject *constant, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON constant from PyString pystr. - constant is the Python string that was found - ("NaN", "Infinity", "-Infinity"). - idx is the index of the first character of the constant - *next_idx_ptr is a return-by-reference index to the first character after - the constant. - - Returns the result of parse_constant - */ - PyObject *rval; - if (s->parse_constant == Py_None) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx); - return NULL; - } - - /* rval = parse_constant(constant) */ - rval = PyObject_CallOneArg(s->parse_constant, constant); - idx += PyString_GET_SIZE(constant); - *next_idx_ptr = idx; - return rval; -} - -#if PY_MAJOR_VERSION < 3 -static PyObject * -_match_number_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON number from PyString pystr. - idx is the index of the first character of the number - *next_idx_ptr is a return-by-reference index to the first character after - the number. - - Returns a new PyObject representation of that number: - PyInt, PyLong, or PyFloat. - May return other types if parse_int or parse_float are set - */ - char *str = PyString_AS_STRING(pystr); - Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1; - Py_ssize_t idx = start; - int is_float = 0; - PyObject *rval; - PyObject *numstr; - - /* read a sign if it's there, make sure it's not the end of the string */ - if (str[idx] == '-') { - if (idx >= end_idx) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, start); - return NULL; - } - idx++; - } - - /* read as many integer digits as we find as long as it doesn't start with 0 */ - if (str[idx] >= '1' && str[idx] <= '9') { - idx++; - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; - } - /* if it starts with 0 we only expect one integer digit */ - else if (str[idx] == '0') { - idx++; - } - /* no integer digits, error */ - else { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, start); - return NULL; - } - - /* if the next char is '.' followed by a digit then read all float digits */ - if (idx < end_idx && str[idx] == '.' && str[idx + 1] >= '0' && str[idx + 1] <= '9') { - is_float = 1; - idx += 2; - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; - } - - /* if the next char is 'e' or 'E' then maybe read the exponent (or backtrack) */ - if (idx < end_idx && (str[idx] == 'e' || str[idx] == 'E')) { - - /* save the index of the 'e' or 'E' just in case we need to backtrack */ - Py_ssize_t e_start = idx; - idx++; - - /* read an exponent sign if present */ - if (idx < end_idx && (str[idx] == '-' || str[idx] == '+')) idx++; - - /* read all digits */ - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; - - /* if we got a digit, then parse as float. if not, backtrack */ - if (str[idx - 1] >= '0' && str[idx - 1] <= '9') { - is_float = 1; - } - else { - idx = e_start; - } - } - - /* copy the section we determined to be a number */ - numstr = PyString_FromStringAndSize(&str[start], idx - start); - if (numstr == NULL) - return NULL; - if (is_float) { - /* parse as a float using a fast path if available, otherwise call user defined method */ - if (s->parse_float != (PyObject *)&PyFloat_Type) { - rval = PyObject_CallOneArg(s->parse_float, numstr); - } - else { - /* rval = PyFloat_FromDouble(PyOS_ascii_atof(PyString_AS_STRING(numstr))); */ - double d = PyOS_string_to_double(PyString_AS_STRING(numstr), - NULL, NULL); - if (d == -1.0 && PyErr_Occurred()) { - Py_DECREF(numstr); - return NULL; - } - rval = PyFloat_FromDouble(d); - } - } - else { - /* parse as an int using a fast path if available, otherwise call user defined method */ - if (s->parse_int != (PyObject *)&PyInt_Type) { - rval = PyObject_CallOneArg(s->parse_int, numstr); - } - else { - rval = PyInt_FromString(PyString_AS_STRING(numstr), NULL, 10); - } - } - Py_DECREF(numstr); - *next_idx_ptr = idx; - return rval; -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -_match_number_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON number from PyUnicode pystr. - idx is the index of the first character of the number - *next_idx_ptr is a return-by-reference index to the first character after - the number. - - Returns a new PyObject representation of that number: - PyInt, PyLong, or PyFloat. - May return other types if parse_int or parse_float are set - */ - PY2_UNUSED int kind = PyUnicode_KIND(pystr); - void *str = PyUnicode_DATA(pystr); - Py_ssize_t end_idx = PyUnicode_GET_LENGTH(pystr) - 1; - Py_ssize_t idx = start; - int is_float = 0; - JSON_UNICHR c; - PyObject *rval; - PyObject *numstr; - - /* read a sign if it's there, make sure it's not the end of the string */ - if (PyUnicode_READ(kind, str, idx) == '-') { - if (idx >= end_idx) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, start); - return NULL; - } - idx++; - } - - /* read as many integer digits as we find as long as it doesn't start with 0 */ - c = PyUnicode_READ(kind, str, idx); - if (c == '0') { - /* if it starts with 0 we only expect one integer digit */ - idx++; - } - else if (IS_DIGIT(c)) { - idx++; - while (idx <= end_idx && IS_DIGIT(PyUnicode_READ(kind, str, idx))) { - idx++; - } - } - else { - /* no integer digits, error */ - raise_errmsg(ERR_EXPECTING_VALUE, pystr, start); - return NULL; - } - - /* if the next char is '.' followed by a digit then read all float digits */ - if (idx < end_idx && - PyUnicode_READ(kind, str, idx) == '.' && - IS_DIGIT(PyUnicode_READ(kind, str, idx + 1))) { - is_float = 1; - idx += 2; - while (idx <= end_idx && IS_DIGIT(PyUnicode_READ(kind, str, idx))) idx++; - } - - /* if the next char is 'e' or 'E' then maybe read the exponent (or backtrack) */ - if (idx < end_idx && - (PyUnicode_READ(kind, str, idx) == 'e' || - PyUnicode_READ(kind, str, idx) == 'E')) { - Py_ssize_t e_start = idx; - idx++; - - /* read an exponent sign if present */ - if (idx < end_idx && - (PyUnicode_READ(kind, str, idx) == '-' || - PyUnicode_READ(kind, str, idx) == '+')) idx++; - - /* read all digits */ - while (idx <= end_idx && IS_DIGIT(PyUnicode_READ(kind, str, idx))) idx++; - - /* if we got a digit, then parse as float. if not, backtrack */ - if (IS_DIGIT(PyUnicode_READ(kind, str, idx - 1))) { - is_float = 1; - } - else { - idx = e_start; - } - } - - /* copy the section we determined to be a number */ -#if PY_MAJOR_VERSION >= 3 - numstr = PyUnicode_Substring(pystr, start, idx); -#else - numstr = PyUnicode_FromUnicode(&((Py_UNICODE *)str)[start], idx - start); -#endif - if (numstr == NULL) - return NULL; - if (is_float) { - /* parse as a float using a fast path if available, otherwise call user defined method */ - if (s->parse_float != (PyObject *)&PyFloat_Type) { - rval = PyObject_CallOneArg(s->parse_float, numstr); - } - else { -#if PY_MAJOR_VERSION >= 3 - rval = PyFloat_FromString(numstr); -#else - rval = PyFloat_FromString(numstr, NULL); -#endif - } - } - else { - /* no fast path for unicode -> int, just call */ - rval = PyObject_CallOneArg(s->parse_int, numstr); - } - Py_DECREF(numstr); - *next_idx_ptr = idx; - return rval; -} - -#if PY_MAJOR_VERSION < 3 -static PyObject * -scan_once_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read one JSON term (of any kind) from PyString pystr. - idx is the index of the first character of the term - *next_idx_ptr is a return-by-reference index to the first character after - the number. - - Returns a new PyObject representation of the term. - */ - char *str = PyString_AS_STRING(pystr); - Py_ssize_t length = PyString_GET_SIZE(pystr); - PyObject *rval = NULL; - int fallthrough = 0; - if (idx < 0 || idx >= length) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx); - return NULL; - } - switch (str[idx]) { - case '"': - /* string */ - rval = scanstring_str(pystr, idx + 1, - PyString_AS_STRING(s->encoding), - s->strict, - next_idx_ptr); - break; - case '{': - /* object */ - if (Py_EnterRecursiveCall(" while decoding a JSON object " - "from a string")) - return NULL; - rval = _parse_object_str(s, pystr, idx + 1, next_idx_ptr); - Py_LeaveRecursiveCall(); - break; - case '[': - /* array */ - if (Py_EnterRecursiveCall(" while decoding a JSON array " - "from a string")) - return NULL; - rval = _parse_array_str(s, pystr, idx + 1, next_idx_ptr); - Py_LeaveRecursiveCall(); - break; - case 'n': - /* null */ - if ((idx + 3 < length) && str[idx + 1] == 'u' && str[idx + 2] == 'l' && str[idx + 3] == 'l') { - Py_INCREF(Py_None); - *next_idx_ptr = idx + 4; - rval = Py_None; - } - else - fallthrough = 1; - break; - case 't': - /* true */ - if ((idx + 3 < length) && str[idx + 1] == 'r' && str[idx + 2] == 'u' && str[idx + 3] == 'e') { - Py_INCREF(Py_True); - *next_idx_ptr = idx + 4; - rval = Py_True; - } - else - fallthrough = 1; - break; - case 'f': - /* false */ - if ((idx + 4 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'l' && str[idx + 3] == 's' && str[idx + 4] == 'e') { - Py_INCREF(Py_False); - *next_idx_ptr = idx + 5; - rval = Py_False; - } - else - fallthrough = 1; - break; - case 'N': - /* NaN */ - if ((idx + 2 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'N') { - rval = _parse_constant(s, pystr, JSON_NaN, idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - case 'I': - /* Infinity */ - if ((idx + 7 < length) && str[idx + 1] == 'n' && str[idx + 2] == 'f' && str[idx + 3] == 'i' && str[idx + 4] == 'n' && str[idx + 5] == 'i' && str[idx + 6] == 't' && str[idx + 7] == 'y') { - rval = _parse_constant(s, pystr, JSON_Infinity, idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - case '-': - /* -Infinity */ - if ((idx + 8 < length) && str[idx + 1] == 'I' && str[idx + 2] == 'n' && str[idx + 3] == 'f' && str[idx + 4] == 'i' && str[idx + 5] == 'n' && str[idx + 6] == 'i' && str[idx + 7] == 't' && str[idx + 8] == 'y') { - rval = _parse_constant(s, pystr, JSON_NegInfinity, idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - default: - fallthrough = 1; - } - /* Didn't find a string, object, array, or named constant. Look for a number. */ - if (fallthrough) - rval = _match_number_str(s, pystr, idx, next_idx_ptr); - return rval; -} -#endif /* PY_MAJOR_VERSION < 3 */ - - -static PyObject * -scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read one JSON term (of any kind) from PyUnicode pystr. - idx is the index of the first character of the term - *next_idx_ptr is a return-by-reference index to the first character after - the number. - - Returns a new PyObject representation of the term. - */ - PY2_UNUSED int kind = PyUnicode_KIND(pystr); - void *str = PyUnicode_DATA(pystr); - Py_ssize_t length = PyUnicode_GET_LENGTH(pystr); - PyObject *rval = NULL; - int fallthrough = 0; - if (idx < 0 || idx >= length) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx); - return NULL; - } - switch (PyUnicode_READ(kind, str, idx)) { - case '"': - /* string */ - rval = scanstring_unicode(pystr, idx + 1, - s->strict, - next_idx_ptr); - break; - case '{': - /* object */ - if (Py_EnterRecursiveCall(" while decoding a JSON object " - "from a unicode string")) - return NULL; - rval = _parse_object_unicode(s, pystr, idx + 1, next_idx_ptr); - Py_LeaveRecursiveCall(); - break; - case '[': - /* array */ - if (Py_EnterRecursiveCall(" while decoding a JSON array " - "from a unicode string")) - return NULL; - rval = _parse_array_unicode(s, pystr, idx + 1, next_idx_ptr); - Py_LeaveRecursiveCall(); - break; - case 'n': - /* null */ - if ((idx + 3 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'u' && - PyUnicode_READ(kind, str, idx + 2) == 'l' && - PyUnicode_READ(kind, str, idx + 3) == 'l') { - Py_INCREF(Py_None); - *next_idx_ptr = idx + 4; - rval = Py_None; - } - else - fallthrough = 1; - break; - case 't': - /* true */ - if ((idx + 3 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'r' && - PyUnicode_READ(kind, str, idx + 2) == 'u' && - PyUnicode_READ(kind, str, idx + 3) == 'e') { - Py_INCREF(Py_True); - *next_idx_ptr = idx + 4; - rval = Py_True; - } - else - fallthrough = 1; - break; - case 'f': - /* false */ - if ((idx + 4 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'a' && - PyUnicode_READ(kind, str, idx + 2) == 'l' && - PyUnicode_READ(kind, str, idx + 3) == 's' && - PyUnicode_READ(kind, str, idx + 4) == 'e') { - Py_INCREF(Py_False); - *next_idx_ptr = idx + 5; - rval = Py_False; - } - else - fallthrough = 1; - break; - case 'N': - /* NaN */ - if ((idx + 2 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'a' && - PyUnicode_READ(kind, str, idx + 2) == 'N') { - rval = _parse_constant(s, pystr, JSON_NaN, idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - case 'I': - /* Infinity */ - if ((idx + 7 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'n' && - PyUnicode_READ(kind, str, idx + 2) == 'f' && - PyUnicode_READ(kind, str, idx + 3) == 'i' && - PyUnicode_READ(kind, str, idx + 4) == 'n' && - PyUnicode_READ(kind, str, idx + 5) == 'i' && - PyUnicode_READ(kind, str, idx + 6) == 't' && - PyUnicode_READ(kind, str, idx + 7) == 'y') { - rval = _parse_constant(s, pystr, JSON_Infinity, idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - case '-': - /* -Infinity */ - if ((idx + 8 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'I' && - PyUnicode_READ(kind, str, idx + 2) == 'n' && - PyUnicode_READ(kind, str, idx + 3) == 'f' && - PyUnicode_READ(kind, str, idx + 4) == 'i' && - PyUnicode_READ(kind, str, idx + 5) == 'n' && - PyUnicode_READ(kind, str, idx + 6) == 'i' && - PyUnicode_READ(kind, str, idx + 7) == 't' && - PyUnicode_READ(kind, str, idx + 8) == 'y') { - rval = _parse_constant(s, pystr, JSON_NegInfinity, idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - default: - fallthrough = 1; - } - /* Didn't find a string, object, array, or named constant. Look for a number. */ - if (fallthrough) - rval = _match_number_unicode(s, pystr, idx, next_idx_ptr); - return rval; -} - -static PyObject * -scanner_call(PyObject *self, PyObject *args, PyObject *kwds) -{ - /* Python callable interface to scan_once_{str,unicode} */ - PyObject *pystr; - PyObject *rval; - Py_ssize_t idx; - Py_ssize_t next_idx = -1; - static char *kwlist[] = {"string", "idx", NULL}; - PyScannerObject *s; - assert(PyScanner_Check(self)); - s = (PyScannerObject *)self; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&:scan_once", kwlist, &pystr, _convertPyInt_AsSsize_t, &idx)) - return NULL; - - if (PyUnicode_Check(pystr)) { - if (PyUnicode_READY(pystr)) - return NULL; - rval = scan_once_unicode(s, pystr, idx, &next_idx); - } -#if PY_MAJOR_VERSION < 3 - else if (PyString_Check(pystr)) { - rval = scan_once_str(s, pystr, idx, &next_idx); - } -#endif /* PY_MAJOR_VERSION < 3 */ - else { - PyErr_Format(PyExc_TypeError, - "first argument must be a string, not %.80s", - Py_TYPE(pystr)->tp_name); - return NULL; - } - PyDict_Clear(s->memo); - return _build_rval_index_tuple(rval, next_idx); -} - -static PyObject * -JSON_ParseEncoding(PyObject *encoding) -{ - if (encoding == Py_None) - return JSON_InternFromString(DEFAULT_ENCODING); -#if PY_MAJOR_VERSION >= 3 - if (PyUnicode_Check(encoding)) { - if (PyUnicode_AsUTF8(encoding) == NULL) { - return NULL; - } - Py_INCREF(encoding); - return encoding; - } -#else /* PY_MAJOR_VERSION >= 3 */ - if (PyString_Check(encoding)) { - Py_INCREF(encoding); - return encoding; - } - if (PyUnicode_Check(encoding)) - return PyUnicode_AsEncodedString(encoding, NULL, NULL); -#endif /* PY_MAJOR_VERSION >= 3 */ - PyErr_SetString(PyExc_TypeError, "encoding must be a string"); - return NULL; -} - -static PyObject * -scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - /* Initialize Scanner object */ - PyObject *ctx; - static char *kwlist[] = {"context", NULL}; - PyScannerObject *s; - PyObject *encoding; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:make_scanner", kwlist, &ctx)) - return NULL; - - s = (PyScannerObject *)type->tp_alloc(type, 0); - if (s == NULL) - return NULL; - - if (s->memo == NULL) { - s->memo = PyDict_New(); - if (s->memo == NULL) - goto bail; - } - - encoding = PyObject_GetAttrString(ctx, "encoding"); - if (encoding == NULL) - goto bail; - s->encoding = JSON_ParseEncoding(encoding); - Py_XDECREF(encoding); - if (s->encoding == NULL) - goto bail; - - /* All of these will fail "gracefully" so we don't need to verify them */ - s->strict_bool = PyObject_GetAttrString(ctx, "strict"); - if (s->strict_bool == NULL) - goto bail; - s->strict = PyObject_IsTrue(s->strict_bool); - if (s->strict < 0) - goto bail; - s->object_hook = PyObject_GetAttrString(ctx, "object_hook"); - if (s->object_hook == NULL) - goto bail; - s->pairs_hook = PyObject_GetAttrString(ctx, "object_pairs_hook"); - if (s->pairs_hook == NULL) - goto bail; - s->parse_float = PyObject_GetAttrString(ctx, "parse_float"); - if (s->parse_float == NULL) - goto bail; - s->parse_int = PyObject_GetAttrString(ctx, "parse_int"); - if (s->parse_int == NULL) - goto bail; - s->parse_constant = PyObject_GetAttrString(ctx, "parse_constant"); - if (s->parse_constant == NULL) - goto bail; - - return (PyObject *)s; - -bail: - Py_DECREF(s); - return NULL; -} - -PyDoc_STRVAR(scanner_doc, "JSON scanner object"); - -static -PyTypeObject PyScannerType = { - PyVarObject_HEAD_INIT(NULL, 0) - "simplejson._speedups.Scanner", /* tp_name */ - sizeof(PyScannerObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - scanner_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - scanner_call, /* tp_call */ - 0, /* tp_str */ - 0,/* PyObject_GenericGetAttr, */ /* tp_getattro */ - 0,/* PyObject_GenericSetAttr, */ /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - scanner_doc, /* tp_doc */ - scanner_traverse, /* tp_traverse */ - scanner_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - scanner_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0,/* PyType_GenericAlloc, */ /* tp_alloc */ - scanner_new, /* tp_new */ - 0,/* PyObject_GC_Del, */ /* tp_free */ -}; - -static PyObject * -encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = { - "markers", - "default", - "encoder", - "indent", - "key_separator", - "item_separator", - "sort_keys", - "skipkeys", - "allow_nan", - "key_memo", - "use_decimal", - "namedtuple_as_object", - "tuple_as_array", - "int_as_string_bitcount", - "item_sort_key", - "encoding", - "for_json", - "ignore_nan", - "Decimal", - "iterable_as_array", - NULL}; - - PyEncoderObject *s; - PyObject *markers, *defaultfn, *encoder, *indent, *key_separator; - PyObject *item_separator, *sort_keys, *skipkeys, *allow_nan, *key_memo; - PyObject *use_decimal, *namedtuple_as_object, *tuple_as_array, *iterable_as_array; - PyObject *int_as_string_bitcount, *item_sort_key, *encoding, *for_json; - PyObject *ignore_nan, *Decimal; - int is_true; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOOOOOOOOOOOOOOOOOOO:make_encoder", kwlist, - &markers, &defaultfn, &encoder, &indent, &key_separator, &item_separator, - &sort_keys, &skipkeys, &allow_nan, &key_memo, &use_decimal, - &namedtuple_as_object, &tuple_as_array, - &int_as_string_bitcount, &item_sort_key, &encoding, &for_json, - &ignore_nan, &Decimal, &iterable_as_array)) - return NULL; - - s = (PyEncoderObject *)type->tp_alloc(type, 0); - if (s == NULL) - return NULL; - - Py_INCREF(markers); - s->markers = markers; - Py_INCREF(defaultfn); - s->defaultfn = defaultfn; - Py_INCREF(encoder); - s->encoder = encoder; -#if PY_MAJOR_VERSION >= 3 - if (encoding == Py_None) { - s->encoding = NULL; - } - else -#endif /* PY_MAJOR_VERSION >= 3 */ - { - s->encoding = JSON_ParseEncoding(encoding); - if (s->encoding == NULL) - goto bail; - } - Py_INCREF(indent); - s->indent = indent; - Py_INCREF(key_separator); - s->key_separator = key_separator; - Py_INCREF(item_separator); - s->item_separator = item_separator; - Py_INCREF(skipkeys); - s->skipkeys_bool = skipkeys; - s->skipkeys = PyObject_IsTrue(skipkeys); - if (s->skipkeys < 0) - goto bail; - Py_INCREF(key_memo); - s->key_memo = key_memo; - s->fast_encode = (PyCFunction_Check(s->encoder) && PyCFunction_GetFunction(s->encoder) == (PyCFunction)py_encode_basestring_ascii); - is_true = PyObject_IsTrue(ignore_nan); - if (is_true < 0) - goto bail; - s->allow_or_ignore_nan = is_true ? JSON_IGNORE_NAN : 0; - is_true = PyObject_IsTrue(allow_nan); - if (is_true < 0) - goto bail; - s->allow_or_ignore_nan |= is_true ? JSON_ALLOW_NAN : 0; - s->use_decimal = PyObject_IsTrue(use_decimal); - if (s->use_decimal < 0) - goto bail; - s->namedtuple_as_object = PyObject_IsTrue(namedtuple_as_object); - if (s->namedtuple_as_object < 0) - goto bail; - s->tuple_as_array = PyObject_IsTrue(tuple_as_array); - if (s->tuple_as_array < 0) - goto bail; - s->iterable_as_array = PyObject_IsTrue(iterable_as_array); - if (s->iterable_as_array < 0) - goto bail; - if (PyInt_Check(int_as_string_bitcount) || PyLong_Check(int_as_string_bitcount)) { - static const unsigned long long_long_bitsize = SIZEOF_LONG_LONG * 8; - long int_as_string_bitcount_val = PyLong_AsLong(int_as_string_bitcount); - if (int_as_string_bitcount_val > 0 && int_as_string_bitcount_val < (long)long_long_bitsize) { - s->max_long_size = PyLong_FromUnsignedLongLong(1ULL << (int)int_as_string_bitcount_val); - s->min_long_size = PyLong_FromLongLong(-1LL << (int)int_as_string_bitcount_val); - if (s->min_long_size == NULL || s->max_long_size == NULL) { - goto bail; - } - } - else { - PyErr_Format(PyExc_TypeError, - "int_as_string_bitcount (%ld) must be greater than 0 and less than the number of bits of a `long long` type (%lu bits)", - int_as_string_bitcount_val, long_long_bitsize); - goto bail; - } - } - else if (int_as_string_bitcount == Py_None) { - Py_INCREF(Py_None); - s->max_long_size = Py_None; - Py_INCREF(Py_None); - s->min_long_size = Py_None; - } - else { - PyErr_SetString(PyExc_TypeError, "int_as_string_bitcount must be None or an integer"); - goto bail; - } - if (item_sort_key != Py_None) { - if (!PyCallable_Check(item_sort_key)) { - PyErr_SetString(PyExc_TypeError, "item_sort_key must be None or callable"); - goto bail; - } - } - else { - is_true = PyObject_IsTrue(sort_keys); - if (is_true < 0) - goto bail; - if (is_true) { - static PyObject *itemgetter0 = NULL; - if (!itemgetter0) { - PyObject *operator = PyImport_ImportModule("operator"); - if (!operator) - goto bail; - itemgetter0 = PyObject_CallMethod(operator, "itemgetter", "i", 0); - Py_DECREF(operator); - } - item_sort_key = itemgetter0; - if (!item_sort_key) - goto bail; - } - } - if (item_sort_key == Py_None) { - Py_INCREF(Py_None); - s->item_sort_kw = Py_None; - } - else { - s->item_sort_kw = PyDict_New(); - if (s->item_sort_kw == NULL) - goto bail; - if (PyDict_SetItemString(s->item_sort_kw, "key", item_sort_key)) - goto bail; - } - Py_INCREF(sort_keys); - s->sort_keys = sort_keys; - Py_INCREF(item_sort_key); - s->item_sort_key = item_sort_key; - Py_INCREF(Decimal); - s->Decimal = Decimal; - s->for_json = PyObject_IsTrue(for_json); - if (s->for_json < 0) - goto bail; - - return (PyObject *)s; - -bail: - Py_DECREF(s); - return NULL; -} - -static PyObject * -encoder_call(PyObject *self, PyObject *args, PyObject *kwds) -{ - /* Python callable interface to encode_listencode_obj */ - static char *kwlist[] = {"obj", "_current_indent_level", NULL}; - PyObject *obj; - Py_ssize_t indent_level; - PyEncoderObject *s; - JSON_Accu rval; - assert(PyEncoder_Check(self)); - s = (PyEncoderObject *)self; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&:_iterencode", kwlist, - &obj, _convertPyInt_AsSsize_t, &indent_level)) - return NULL; - if (JSON_Accu_Init(&rval)) - return NULL; - if (encoder_listencode_obj(s, &rval, obj, indent_level)) { - JSON_Accu_Destroy(&rval); - return NULL; - } - return JSON_Accu_FinishAsList(&rval); -} - -static PyObject * -_encoded_const(PyObject *obj) -{ - /* Return the JSON string representation of None, True, False */ - if (obj == Py_None) { - static PyObject *s_null = NULL; - if (s_null == NULL) { - s_null = JSON_InternFromString("null"); - } - Py_INCREF(s_null); - return s_null; - } - else if (obj == Py_True) { - static PyObject *s_true = NULL; - if (s_true == NULL) { - s_true = JSON_InternFromString("true"); - } - Py_INCREF(s_true); - return s_true; - } - else if (obj == Py_False) { - static PyObject *s_false = NULL; - if (s_false == NULL) { - s_false = JSON_InternFromString("false"); - } - Py_INCREF(s_false); - return s_false; - } - else { - PyErr_SetString(PyExc_ValueError, "not a const"); - return NULL; - } -} - -static PyObject * -encoder_encode_float(PyEncoderObject *s, PyObject *obj) -{ - /* Return the JSON representation of a PyFloat */ - double i = PyFloat_AS_DOUBLE(obj); - if (!Py_IS_FINITE(i)) { - if (!s->allow_or_ignore_nan) { - PyErr_SetString(PyExc_ValueError, "Out of range float values are not JSON compliant"); - return NULL; - } - if (s->allow_or_ignore_nan & JSON_IGNORE_NAN) { - return _encoded_const(Py_None); - } - /* JSON_ALLOW_NAN is set */ - else if (i > 0) { - Py_INCREF(JSON_Infinity); - return JSON_Infinity; - } - else if (i < 0) { - Py_INCREF(JSON_NegInfinity); - return JSON_NegInfinity; - } - else { - Py_INCREF(JSON_NaN); - return JSON_NaN; - } - } - /* Use a better float format here? */ - if (PyFloat_CheckExact(obj)) { - return PyObject_Repr(obj); - } - else { - /* See #118, do not trust custom str/repr */ - PyObject *res; - PyObject *tmp = PyObject_CallOneArg((PyObject *)&PyFloat_Type, obj); - if (tmp == NULL) { - return NULL; - } - res = PyObject_Repr(tmp); - Py_DECREF(tmp); - return res; - } -} - -static PyObject * -encoder_encode_string(PyEncoderObject *s, PyObject *obj) -{ - /* Return the JSON representation of a string */ - PyObject *encoded; - - if (s->fast_encode) { - return py_encode_basestring_ascii(NULL, obj); - } - encoded = PyObject_CallOneArg(s->encoder, obj); - if (encoded != NULL && -#if PY_MAJOR_VERSION < 3 - !PyString_Check(encoded) && -#endif /* PY_MAJOR_VERSION < 3 */ - !PyUnicode_Check(encoded)) - { - PyErr_Format(PyExc_TypeError, - "encoder() must return a string, not %.80s", - Py_TYPE(encoded)->tp_name); - Py_DECREF(encoded); - return NULL; - } - return encoded; -} - -static int -_steal_accumulate(JSON_Accu *accu, PyObject *stolen) -{ - /* Append stolen and then decrement its reference count */ - int rval = JSON_Accu_Accumulate(accu, stolen); - Py_DECREF(stolen); - return rval; -} - -static int -encoder_listencode_obj(PyEncoderObject *s, JSON_Accu *rval, PyObject *obj, Py_ssize_t indent_level) -{ - /* Encode Python object obj to a JSON term, rval is a PyList */ - int rv = -1; - do { - PyObject *newobj; - if (obj == Py_None || obj == Py_True || obj == Py_False) { - PyObject *cstr = _encoded_const(obj); - if (cstr != NULL) - rv = _steal_accumulate(rval, cstr); - } - else if ((PyBytes_Check(obj) && s->encoding != NULL) || - PyUnicode_Check(obj)) - { - PyObject *encoded = encoder_encode_string(s, obj); - if (encoded != NULL) - rv = _steal_accumulate(rval, encoded); - } - else if (PyInt_Check(obj) || PyLong_Check(obj)) { - PyObject *encoded; - if (PyInt_CheckExact(obj) || PyLong_CheckExact(obj)) { - encoded = PyObject_Str(obj); - } - else { - /* See #118, do not trust custom str/repr */ - PyObject *tmp = PyObject_CallOneArg((PyObject *)&PyLong_Type, obj); - if (tmp == NULL) { - encoded = NULL; - } - else { - encoded = PyObject_Str(tmp); - Py_DECREF(tmp); - } - } - if (encoded != NULL) { - encoded = maybe_quote_bigint(s, encoded, obj); - if (encoded == NULL) - break; - rv = _steal_accumulate(rval, encoded); - } - } - else if (PyFloat_Check(obj)) { - PyObject *encoded = encoder_encode_float(s, obj); - if (encoded != NULL) - rv = _steal_accumulate(rval, encoded); - } - else if (s->for_json && _call_json_method(obj, FOR_JSON_METHOD_NAME, &newobj)) { - if (newobj == NULL) { - return -1; - } - if (Py_EnterRecursiveCall(" while encoding a JSON object")) { - Py_DECREF(newobj); - return rv; - } - rv = encoder_listencode_obj(s, rval, newobj, indent_level); - Py_DECREF(newobj); - Py_LeaveRecursiveCall(); - } - else if (s->namedtuple_as_object && _call_json_method(obj, ASDICT_METHOD_NAME, &newobj)) { - if (newobj == NULL) { - return -1; - } - if (Py_EnterRecursiveCall(" while encoding a JSON object")) { - Py_DECREF(newobj); - return rv; - } - if (PyDict_Check(newobj)) { - rv = encoder_listencode_dict(s, rval, newobj, indent_level); - } else { - PyErr_Format( - PyExc_TypeError, - "_asdict() must return a dict, not %.80s", - Py_TYPE(newobj)->tp_name - ); - rv = -1; - } - Py_DECREF(newobj); - Py_LeaveRecursiveCall(); - } - else if (PyList_Check(obj) || (s->tuple_as_array && PyTuple_Check(obj))) { - if (Py_EnterRecursiveCall(" while encoding a JSON object")) - return rv; - rv = encoder_listencode_list(s, rval, obj, indent_level); - Py_LeaveRecursiveCall(); - } - else if (PyDict_Check(obj)) { - if (Py_EnterRecursiveCall(" while encoding a JSON object")) - return rv; - rv = encoder_listencode_dict(s, rval, obj, indent_level); - Py_LeaveRecursiveCall(); - } - else if (s->use_decimal && PyObject_TypeCheck(obj, (PyTypeObject *)s->Decimal)) { - PyObject *encoded = PyObject_Str(obj); - if (encoded != NULL) - rv = _steal_accumulate(rval, encoded); - } - else if (is_raw_json(obj)) - { - PyObject *encoded = PyObject_GetAttrString(obj, "encoded_json"); - if (encoded != NULL) - rv = _steal_accumulate(rval, encoded); - } - else { - PyObject *ident = NULL; - PyObject *newobj; - if (s->iterable_as_array) { - newobj = PyObject_GetIter(obj); - if (newobj == NULL) - PyErr_Clear(); - else { - rv = encoder_listencode_list(s, rval, newobj, indent_level); - Py_DECREF(newobj); - break; - } - } - if (s->markers != Py_None) { - int has_key; - ident = PyLong_FromVoidPtr(obj); - if (ident == NULL) - break; - has_key = PyDict_Contains(s->markers, ident); - if (has_key) { - if (has_key != -1) - PyErr_SetString(PyExc_ValueError, "Circular reference detected"); - Py_DECREF(ident); - break; - } - if (PyDict_SetItem(s->markers, ident, obj)) { - Py_DECREF(ident); - break; - } - } - if (Py_EnterRecursiveCall(" while encoding a JSON object")) - return rv; - newobj = PyObject_CallOneArg(s->defaultfn, obj); - if (newobj == NULL) { - Py_XDECREF(ident); - Py_LeaveRecursiveCall(); - break; - } - rv = encoder_listencode_obj(s, rval, newobj, indent_level); - Py_LeaveRecursiveCall(); - Py_DECREF(newobj); - if (rv) { - Py_XDECREF(ident); - rv = -1; - } - else if (ident != NULL) { - if (PyDict_DelItem(s->markers, ident)) { - Py_XDECREF(ident); - rv = -1; - } - Py_XDECREF(ident); - } - } - } while (0); - return rv; -} - -static int -encoder_listencode_dict(PyEncoderObject *s, JSON_Accu *rval, PyObject *dct, Py_ssize_t indent_level) -{ - /* Encode Python dict dct a JSON term */ - static PyObject *open_dict = NULL; - static PyObject *close_dict = NULL; - static PyObject *empty_dict = NULL; - PyObject *kstr = NULL; - PyObject *ident = NULL; - PyObject *iter = NULL; - PyObject *item = NULL; - PyObject *items = NULL; - PyObject *encoded = NULL; - Py_ssize_t idx; - - if (open_dict == NULL || close_dict == NULL || empty_dict == NULL) { - open_dict = JSON_InternFromString("{"); - close_dict = JSON_InternFromString("}"); - empty_dict = JSON_InternFromString("{}"); - if (open_dict == NULL || close_dict == NULL || empty_dict == NULL) - return -1; - } - if (PyDict_Size(dct) == 0) - return JSON_Accu_Accumulate(rval, empty_dict); - - if (s->markers != Py_None) { - int has_key; - ident = PyLong_FromVoidPtr(dct); - if (ident == NULL) - goto bail; - has_key = PyDict_Contains(s->markers, ident); - if (has_key) { - if (has_key != -1) - PyErr_SetString(PyExc_ValueError, "Circular reference detected"); - goto bail; - } - if (PyDict_SetItem(s->markers, ident, dct)) { - goto bail; - } - } - - if (JSON_Accu_Accumulate(rval, open_dict)) - goto bail; - - if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level += 1; - /* - newline_indent = '\n' + (_indent * _current_indent_level) - separator = _item_separator + newline_indent - buf += newline_indent - */ - } - - iter = encoder_dict_iteritems(s, dct); - if (iter == NULL) - goto bail; - - idx = 0; - while ((item = PyIter_Next(iter))) { - PyObject *encoded, *key, *value; - if (!PyTuple_Check(item) || Py_SIZE(item) != 2) { - PyErr_SetString(PyExc_ValueError, "items must return 2-tuples"); - goto bail; - } - key = PyTuple_GET_ITEM(item, 0); - if (key == NULL) - goto bail; - value = PyTuple_GET_ITEM(item, 1); - if (value == NULL) - goto bail; - - kstr = encoder_stringify_key(s, key); - if (kstr == NULL) - goto bail; - else if (kstr == Py_None) { - /* skipkeys */ - Py_DECREF(item); - Py_DECREF(kstr); - continue; - } - if (idx) { - if (JSON_Accu_Accumulate(rval, s->item_separator)) - goto bail; - } - /* - * Only cache the encoding of string keys. False and True are - * indistinguishable from 0 and 1 in a dictionary lookup and there - * may be other quirks with user defined subclasses. - */ - encoded = PyDict_GetItem(s->key_memo, kstr); - if (encoded != NULL) { - Py_INCREF(encoded); - Py_CLEAR(kstr); - } else { - encoded = encoder_encode_string(s, kstr); - Py_CLEAR(kstr); - if (encoded == NULL) - goto bail; - if (PyDict_SetItem(s->key_memo, key, encoded)) - goto bail; - } - if (JSON_Accu_Accumulate(rval, encoded)) { - goto bail; - } - Py_CLEAR(encoded); - if (JSON_Accu_Accumulate(rval, s->key_separator)) - goto bail; - if (encoder_listencode_obj(s, rval, value, indent_level)) - goto bail; - Py_CLEAR(item); - idx += 1; - } - Py_CLEAR(iter); - if (PyErr_Occurred()) - goto bail; - if (ident != NULL) { - if (PyDict_DelItem(s->markers, ident)) - goto bail; - Py_CLEAR(ident); - } - if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level -= 1; - /* - yield '\n' + (_indent * _current_indent_level) - */ - } - if (JSON_Accu_Accumulate(rval, close_dict)) - goto bail; - return 0; - -bail: - Py_XDECREF(encoded); - Py_XDECREF(items); - Py_XDECREF(item); - Py_XDECREF(iter); - Py_XDECREF(kstr); - Py_XDECREF(ident); - return -1; -} - - -static int -encoder_listencode_list(PyEncoderObject *s, JSON_Accu *rval, PyObject *seq, Py_ssize_t indent_level) -{ - /* Encode Python list seq to a JSON term */ - static PyObject *open_array = NULL; - static PyObject *close_array = NULL; - static PyObject *empty_array = NULL; - PyObject *ident = NULL; - PyObject *iter = NULL; - PyObject *obj = NULL; - int is_true; - int i = 0; - - if (open_array == NULL || close_array == NULL || empty_array == NULL) { - open_array = JSON_InternFromString("["); - close_array = JSON_InternFromString("]"); - empty_array = JSON_InternFromString("[]"); - if (open_array == NULL || close_array == NULL || empty_array == NULL) - return -1; - } - ident = NULL; - is_true = PyObject_IsTrue(seq); - if (is_true == -1) - return -1; - else if (is_true == 0) - return JSON_Accu_Accumulate(rval, empty_array); - - if (s->markers != Py_None) { - int has_key; - ident = PyLong_FromVoidPtr(seq); - if (ident == NULL) - goto bail; - has_key = PyDict_Contains(s->markers, ident); - if (has_key) { - if (has_key != -1) - PyErr_SetString(PyExc_ValueError, "Circular reference detected"); - goto bail; - } - if (PyDict_SetItem(s->markers, ident, seq)) { - goto bail; - } - } - - iter = PyObject_GetIter(seq); - if (iter == NULL) - goto bail; - - if (JSON_Accu_Accumulate(rval, open_array)) - goto bail; - if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level += 1; - /* - newline_indent = '\n' + (_indent * _current_indent_level) - separator = _item_separator + newline_indent - buf += newline_indent - */ - } - while ((obj = PyIter_Next(iter))) { - if (i) { - if (JSON_Accu_Accumulate(rval, s->item_separator)) - goto bail; - } - if (encoder_listencode_obj(s, rval, obj, indent_level)) - goto bail; - i++; - Py_CLEAR(obj); - } - Py_CLEAR(iter); - if (PyErr_Occurred()) - goto bail; - if (ident != NULL) { - if (PyDict_DelItem(s->markers, ident)) - goto bail; - Py_CLEAR(ident); - } - if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level -= 1; - /* - yield '\n' + (_indent * _current_indent_level) - */ - } - if (JSON_Accu_Accumulate(rval, close_array)) - goto bail; - return 0; - -bail: - Py_XDECREF(obj); - Py_XDECREF(iter); - Py_XDECREF(ident); - return -1; -} - -static void -encoder_dealloc(PyObject *self) -{ - /* bpo-31095: UnTrack is needed before calling any callbacks */ - PyObject_GC_UnTrack(self); - encoder_clear(self); - Py_TYPE(self)->tp_free(self); -} - -static int -encoder_traverse(PyObject *self, visitproc visit, void *arg) -{ - PyEncoderObject *s; - assert(PyEncoder_Check(self)); - s = (PyEncoderObject *)self; - Py_VISIT(s->markers); - Py_VISIT(s->defaultfn); - Py_VISIT(s->encoder); - Py_VISIT(s->encoding); - Py_VISIT(s->indent); - Py_VISIT(s->key_separator); - Py_VISIT(s->item_separator); - Py_VISIT(s->key_memo); - Py_VISIT(s->sort_keys); - Py_VISIT(s->item_sort_kw); - Py_VISIT(s->item_sort_key); - Py_VISIT(s->max_long_size); - Py_VISIT(s->min_long_size); - Py_VISIT(s->Decimal); - return 0; -} - -static int -encoder_clear(PyObject *self) -{ - /* Deallocate Encoder */ - PyEncoderObject *s; - assert(PyEncoder_Check(self)); - s = (PyEncoderObject *)self; - Py_CLEAR(s->markers); - Py_CLEAR(s->defaultfn); - Py_CLEAR(s->encoder); - Py_CLEAR(s->encoding); - Py_CLEAR(s->indent); - Py_CLEAR(s->key_separator); - Py_CLEAR(s->item_separator); - Py_CLEAR(s->key_memo); - Py_CLEAR(s->skipkeys_bool); - Py_CLEAR(s->sort_keys); - Py_CLEAR(s->item_sort_kw); - Py_CLEAR(s->item_sort_key); - Py_CLEAR(s->max_long_size); - Py_CLEAR(s->min_long_size); - Py_CLEAR(s->Decimal); - return 0; -} - -PyDoc_STRVAR(encoder_doc, "_iterencode(obj, _current_indent_level) -> iterable"); - -static -PyTypeObject PyEncoderType = { - PyVarObject_HEAD_INIT(NULL, 0) - "simplejson._speedups.Encoder", /* tp_name */ - sizeof(PyEncoderObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - encoder_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - encoder_call, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - encoder_doc, /* tp_doc */ - encoder_traverse, /* tp_traverse */ - encoder_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - encoder_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - encoder_new, /* tp_new */ - 0, /* tp_free */ -}; - -static PyMethodDef speedups_methods[] = { - {"encode_basestring_ascii", - (PyCFunction)py_encode_basestring_ascii, - METH_O, - pydoc_encode_basestring_ascii}, - {"scanstring", - (PyCFunction)py_scanstring, - METH_VARARGS, - pydoc_scanstring}, - {NULL, NULL, 0, NULL} -}; - -PyDoc_STRVAR(module_doc, -"simplejson speedups\n"); - -#if PY_MAJOR_VERSION >= 3 -static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - "_speedups", /* m_name */ - module_doc, /* m_doc */ - -1, /* m_size */ - speedups_methods, /* m_methods */ - NULL, /* m_reload */ - NULL, /* m_traverse */ - NULL, /* m_clear*/ - NULL, /* m_free */ -}; -#endif - -PyObject * -import_dependency(char *module_name, char *attr_name) -{ - PyObject *rval; - PyObject *module = PyImport_ImportModule(module_name); - if (module == NULL) - return NULL; - rval = PyObject_GetAttrString(module, attr_name); - Py_DECREF(module); - return rval; -} - -static int -init_constants(void) -{ - JSON_NaN = JSON_InternFromString("NaN"); - if (JSON_NaN == NULL) - return 0; - JSON_Infinity = JSON_InternFromString("Infinity"); - if (JSON_Infinity == NULL) - return 0; - JSON_NegInfinity = JSON_InternFromString("-Infinity"); - if (JSON_NegInfinity == NULL) - return 0; -#if PY_MAJOR_VERSION >= 3 - JSON_EmptyUnicode = PyUnicode_New(0, 127); -#else /* PY_MAJOR_VERSION >= 3 */ - JSON_EmptyStr = PyString_FromString(""); - if (JSON_EmptyStr == NULL) - return 0; - JSON_EmptyUnicode = PyUnicode_FromUnicode(NULL, 0); -#endif /* PY_MAJOR_VERSION >= 3 */ - if (JSON_EmptyUnicode == NULL) - return 0; - - return 1; -} - -static PyObject * -moduleinit(void) -{ - PyObject *m; - if (PyType_Ready(&PyScannerType) < 0) - return NULL; - if (PyType_Ready(&PyEncoderType) < 0) - return NULL; - if (!init_constants()) - return NULL; - -#if PY_MAJOR_VERSION >= 3 - m = PyModule_Create(&moduledef); -#else - m = Py_InitModule3("_speedups", speedups_methods, module_doc); -#endif - Py_INCREF((PyObject*)&PyScannerType); - PyModule_AddObject(m, "make_scanner", (PyObject*)&PyScannerType); - Py_INCREF((PyObject*)&PyEncoderType); - PyModule_AddObject(m, "make_encoder", (PyObject*)&PyEncoderType); - RawJSONType = import_dependency("simplejson.raw_json", "RawJSON"); - if (RawJSONType == NULL) - return NULL; - JSONDecodeError = import_dependency("simplejson.errors", "JSONDecodeError"); - if (JSONDecodeError == NULL) - return NULL; - return m; -} - -#if PY_MAJOR_VERSION >= 3 -PyMODINIT_FUNC -PyInit__speedups(void) -{ - return moduleinit(); -} -#else -void -init_speedups(void) -{ - moduleinit(); -} -#endif diff --git a/contrib/python/simplejson/py2/simplejson/compat.py b/contrib/python/simplejson/py2/simplejson/compat.py deleted file mode 100644 index 5fc14128443..00000000000 --- a/contrib/python/simplejson/py2/simplejson/compat.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Python 3 compatibility shims -""" -import sys -if sys.version_info[0] < 3: - PY3 = False - def b(s): - return s - try: - from cStringIO import StringIO - except ImportError: - from StringIO import StringIO - BytesIO = StringIO - text_type = unicode - binary_type = str - string_types = (basestring,) - integer_types = (int, long) - unichr = unichr - reload_module = reload -else: - PY3 = True - if sys.version_info[:2] >= (3, 4): - from importlib import reload as reload_module - else: - from imp import reload as reload_module - def b(s): - return bytes(s, 'latin1') - from io import StringIO, BytesIO - text_type = str - binary_type = bytes - string_types = (str,) - integer_types = (int,) - unichr = chr - -long_type = integer_types[-1] diff --git a/contrib/python/simplejson/py2/simplejson/decoder.py b/contrib/python/simplejson/py2/simplejson/decoder.py deleted file mode 100644 index c99a976d84e..00000000000 --- a/contrib/python/simplejson/py2/simplejson/decoder.py +++ /dev/null @@ -1,416 +0,0 @@ -"""Implementation of JSONDecoder -""" -from __future__ import absolute_import -import re -import sys -import struct -from .compat import PY3, unichr -from .scanner import make_scanner, JSONDecodeError - -def _import_c_scanstring(): - try: - from ._speedups import scanstring - return scanstring - except ImportError: - return None -c_scanstring = _import_c_scanstring() - -# NOTE (3.1.0): JSONDecodeError may still be imported from this module for -# compatibility, but it was never in the __all__ -__all__ = ['JSONDecoder'] - -FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL - -def _floatconstants(): - if sys.version_info < (2, 6): - _BYTES = '7FF80000000000007FF0000000000000'.decode('hex') - nan, inf = struct.unpack('>dd', _BYTES) - else: - nan = float('nan') - inf = float('inf') - return nan, inf, -inf - -NaN, PosInf, NegInf = _floatconstants() - -_CONSTANTS = { - '-Infinity': NegInf, - 'Infinity': PosInf, - 'NaN': NaN, -} - -STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS) -BACKSLASH = { - '"': u'"', '\\': u'\\', '/': u'/', - 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t', -} - -DEFAULT_ENCODING = "utf-8" - -if hasattr(sys, 'get_int_max_str_digits'): - bounded_int = int -else: - def bounded_int(s, INT_MAX_STR_DIGITS=4300): - """Backport of the integer string length conversion limitation - - https://docs.python.org/3/library/stdtypes.html#int-max-str-digits - """ - if len(s) > INT_MAX_STR_DIGITS: - raise ValueError("Exceeds the limit (%s) for integer string conversion: value has %s digits" % (INT_MAX_STR_DIGITS, len(s))) - return int(s) - - -def scan_four_digit_hex(s, end, _m=re.compile(r'^[0-9a-fA-F]{4}$').match): - """Scan a four digit hex number from s[end:end + 4] - """ - msg = "Invalid \\uXXXX escape sequence" - esc = s[end:end + 4] - if not _m(esc): - raise JSONDecodeError(msg, s, end - 2) - try: - return int(esc, 16), end + 4 - except ValueError: - raise JSONDecodeError(msg, s, end - 2) - -def py_scanstring(s, end, encoding=None, strict=True, - _b=BACKSLASH, _m=STRINGCHUNK.match, _join=u''.join, - _PY3=PY3, _maxunicode=sys.maxunicode, - _scan_four_digit_hex=scan_four_digit_hex): - """Scan the string s for a JSON string. End is the index of the - character in s after the quote that started the JSON string. - Unescapes all valid JSON string escape sequences and raises ValueError - on attempt to decode an invalid string. If strict is False then literal - control characters are allowed in the string. - - Returns a tuple of the decoded string and the index of the character in s - after the end quote.""" - if encoding is None: - encoding = DEFAULT_ENCODING - chunks = [] - _append = chunks.append - begin = end - 1 - while 1: - chunk = _m(s, end) - if chunk is None: - raise JSONDecodeError( - "Unterminated string starting at", s, begin) - prev_end = end - end = chunk.end() - content, terminator = chunk.groups() - # Content is contains zero or more unescaped string characters - if content: - if not _PY3 and not isinstance(content, unicode): - content = unicode(content, encoding) - _append(content) - # Terminator is the end of string, a literal control character, - # or a backslash denoting that an escape sequence follows - if terminator == '"': - break - elif terminator != '\\': - if strict: - msg = "Invalid control character %r at" - raise JSONDecodeError(msg, s, prev_end) - else: - _append(terminator) - continue - try: - esc = s[end] - except IndexError: - raise JSONDecodeError( - "Unterminated string starting at", s, begin) - # If not a unicode escape sequence, must be in the lookup table - if esc != 'u': - try: - char = _b[esc] - except KeyError: - msg = "Invalid \\X escape sequence %r" - raise JSONDecodeError(msg, s, end) - end += 1 - else: - # Unicode escape sequence - uni, end = _scan_four_digit_hex(s, end + 1) - # Check for surrogate pair on UCS-4 systems - # Note that this will join high/low surrogate pairs - # but will also pass unpaired surrogates through - if (_maxunicode > 65535 and - uni & 0xfc00 == 0xd800 and - s[end:end + 2] == '\\u'): - uni2, end2 = _scan_four_digit_hex(s, end + 2) - if uni2 & 0xfc00 == 0xdc00: - uni = 0x10000 + (((uni - 0xd800) << 10) | - (uni2 - 0xdc00)) - end = end2 - char = unichr(uni) - # Append the unescaped character - _append(char) - return _join(chunks), end - - -# Use speedup if available -scanstring = c_scanstring or py_scanstring - -WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS) -WHITESPACE_STR = ' \t\n\r' - -def JSONObject(state, encoding, strict, scan_once, object_hook, - object_pairs_hook, memo=None, - _w=WHITESPACE.match, _ws=WHITESPACE_STR): - (s, end) = state - # Backwards compatibility - if memo is None: - memo = {} - memo_get = memo.setdefault - pairs = [] - # Use a slice to prevent IndexError from being raised, the following - # check will raise a more specific ValueError if the string is empty - nextchar = s[end:end + 1] - # Normally we expect nextchar == '"' - if nextchar != '"': - if nextchar in _ws: - end = _w(s, end).end() - nextchar = s[end:end + 1] - # Trivial empty object - if nextchar == '}': - if object_pairs_hook is not None: - result = object_pairs_hook(pairs) - return result, end + 1 - pairs = {} - if object_hook is not None: - pairs = object_hook(pairs) - return pairs, end + 1 - elif nextchar != '"': - raise JSONDecodeError( - "Expecting property name enclosed in double quotes or '}'", - s, end) - end += 1 - while True: - key, end = scanstring(s, end, encoding, strict) - key = memo_get(key, key) - - # To skip some function call overhead we optimize the fast paths where - # the JSON key separator is ": " or just ":". - if s[end:end + 1] != ':': - end = _w(s, end).end() - if s[end:end + 1] != ':': - raise JSONDecodeError("Expecting ':' delimiter", s, end) - - end += 1 - - try: - if s[end] in _ws: - end += 1 - if s[end] in _ws: - end = _w(s, end + 1).end() - except IndexError: - pass - - value, end = scan_once(s, end) - pairs.append((key, value)) - - try: - nextchar = s[end] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end] - except IndexError: - nextchar = '' - end += 1 - - if nextchar == '}': - break - elif nextchar != ',': - raise JSONDecodeError("Expecting ',' delimiter or '}'", s, end - 1) - - try: - nextchar = s[end] - if nextchar in _ws: - end += 1 - nextchar = s[end] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end] - except IndexError: - nextchar = '' - - end += 1 - if nextchar != '"': - raise JSONDecodeError( - "Expecting property name enclosed in double quotes", - s, end - 1) - - if object_pairs_hook is not None: - result = object_pairs_hook(pairs) - return result, end - pairs = dict(pairs) - if object_hook is not None: - pairs = object_hook(pairs) - return pairs, end - -def JSONArray(state, scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): - (s, end) = state - values = [] - nextchar = s[end:end + 1] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end:end + 1] - # Look-ahead for trivial empty array - if nextchar == ']': - return values, end + 1 - elif nextchar == '': - raise JSONDecodeError("Expecting value or ']'", s, end) - _append = values.append - while True: - value, end = scan_once(s, end) - _append(value) - nextchar = s[end:end + 1] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end:end + 1] - end += 1 - if nextchar == ']': - break - elif nextchar != ',': - raise JSONDecodeError("Expecting ',' delimiter or ']'", s, end - 1) - - try: - if s[end] in _ws: - end += 1 - if s[end] in _ws: - end = _w(s, end + 1).end() - except IndexError: - pass - - return values, end - -class JSONDecoder(object): - """Simple JSON <http://json.org> decoder - - Performs the following translations in decoding by default: - - +---------------+-------------------+ - | JSON | Python | - +===============+===================+ - | object | dict | - +---------------+-------------------+ - | array | list | - +---------------+-------------------+ - | string | str, unicode | - +---------------+-------------------+ - | number (int) | int, long | - +---------------+-------------------+ - | number (real) | float | - +---------------+-------------------+ - | true | True | - +---------------+-------------------+ - | false | False | - +---------------+-------------------+ - | null | None | - +---------------+-------------------+ - - When allow_nan=True, it also understands - ``NaN``, ``Infinity``, and ``-Infinity`` as - their corresponding ``float`` values, which is outside the JSON spec. - - """ - - def __init__(self, encoding=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, strict=True, - object_pairs_hook=None, allow_nan=False): - """ - *encoding* determines the encoding used to interpret any - :class:`str` objects decoded by this instance (``'utf-8'`` by - default). It has no effect when decoding :class:`unicode` objects. - - Note that currently only encodings that are a superset of ASCII work, - strings of other encodings should be passed in as :class:`unicode`. - - *object_hook*, if specified, will be called with the result of every - JSON object decoded and its return value will be used in place of the - given :class:`dict`. This can be used to provide custom - deserializations (e.g. to support JSON-RPC class hinting). - - *object_pairs_hook* is an optional function that will be called with - the result of any object literal decode with an ordered list of pairs. - The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders - that rely on the order that the key and value pairs are decoded (for - example, :func:`collections.OrderedDict` will remember the order of - insertion). If *object_hook* is also defined, the *object_pairs_hook* - takes priority. - - *parse_float*, if specified, will be called with the string of every - JSON float to be decoded. By default, this is equivalent to - ``float(num_str)``. This can be used to use another datatype or parser - for JSON floats (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every - JSON int to be decoded. By default, this is equivalent to - ``int(num_str)``. This can be used to use another datatype or parser - for JSON integers (e.g. :class:`float`). - - *allow_nan*, if True (default false), will allow the parser to - accept the non-standard floats ``NaN``, ``Infinity``, and ``-Infinity``. - - *parse_constant*, if specified, will be - called with one of the following strings: ``'-Infinity'``, - ``'Infinity'``, ``'NaN'``. It is not recommended to use this feature, - as it is rare to parse non-compliant JSON containing these values. - - *strict* controls the parser's behavior when it encounters an - invalid control character in a string. The default setting of - ``True`` means that unescaped control characters are parse errors, if - ``False`` then control characters will be allowed in strings. - - """ - if encoding is None: - encoding = DEFAULT_ENCODING - self.encoding = encoding - self.object_hook = object_hook - self.object_pairs_hook = object_pairs_hook - self.parse_float = parse_float or float - self.parse_int = parse_int or bounded_int - self.parse_constant = parse_constant or (allow_nan and _CONSTANTS.__getitem__ or None) - self.strict = strict - self.parse_object = JSONObject - self.parse_array = JSONArray - self.parse_string = scanstring - self.memo = {} - self.scan_once = make_scanner(self) - - def decode(self, s, _w=WHITESPACE.match, _PY3=PY3): - """Return the Python representation of ``s`` (a ``str`` or ``unicode`` - instance containing a JSON document) - - """ - if _PY3 and isinstance(s, bytes): - s = str(s, self.encoding) - obj, end = self.raw_decode(s) - end = _w(s, end).end() - if end != len(s): - raise JSONDecodeError("Extra data", s, end, len(s)) - return obj - - def raw_decode(self, s, idx=0, _w=WHITESPACE.match, _PY3=PY3): - """Decode a JSON document from ``s`` (a ``str`` or ``unicode`` - beginning with a JSON document) and return a 2-tuple of the Python - representation and the index in ``s`` where the document ended. - Optionally, ``idx`` can be used to specify an offset in ``s`` where - the JSON document begins. - - This can be used to decode a JSON document from a string that may - have extraneous data at the end. - - """ - if idx < 0: - # Ensure that raw_decode bails on negative indexes, the regex - # would otherwise mask this behavior. #98 - raise JSONDecodeError('Expecting value', s, idx) - if _PY3 and not isinstance(s, str): - raise TypeError("Input string must be text, not bytes") - # strip UTF-8 bom - if len(s) > idx: - ord0 = ord(s[idx]) - if ord0 == 0xfeff: - idx += 1 - elif ord0 == 0xef and s[idx:idx + 3] == '\xef\xbb\xbf': - idx += 3 - return self.scan_once(s, idx=_w(s, idx).end()) diff --git a/contrib/python/simplejson/py2/simplejson/encoder.py b/contrib/python/simplejson/py2/simplejson/encoder.py deleted file mode 100644 index 661ff361bf2..00000000000 --- a/contrib/python/simplejson/py2/simplejson/encoder.py +++ /dev/null @@ -1,740 +0,0 @@ -"""Implementation of JSONEncoder -""" -from __future__ import absolute_import -import re -from operator import itemgetter -# Do not import Decimal directly to avoid reload issues -import decimal -from .compat import binary_type, text_type, string_types, integer_types, PY3 -def _import_speedups(): - try: - from . import _speedups - return _speedups.encode_basestring_ascii, _speedups.make_encoder - except ImportError: - return None, None -c_encode_basestring_ascii, c_make_encoder = _import_speedups() - -from .decoder import PosInf -from .raw_json import RawJSON - -ESCAPE = re.compile(r'[\x00-\x1f\\"]') -ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') -HAS_UTF8 = re.compile(r'[\x80-\xff]') -ESCAPE_DCT = { - '\\': '\\\\', - '"': '\\"', - '\b': '\\b', - '\f': '\\f', - '\n': '\\n', - '\r': '\\r', - '\t': '\\t', -} -for i in range(0x20): - #ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i)) - ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,)) -del i - -FLOAT_REPR = repr - -def encode_basestring(s, _PY3=PY3, _q=u'"'): - """Return a JSON representation of a Python string - - """ - if _PY3: - if isinstance(s, bytes): - s = str(s, 'utf-8') - elif type(s) is not str: - # convert an str subclass instance to exact str - # raise a TypeError otherwise - s = str.__str__(s) - else: - if isinstance(s, str) and HAS_UTF8.search(s) is not None: - s = unicode(s, 'utf-8') - elif type(s) not in (str, unicode): - # convert an str subclass instance to exact str - # convert a unicode subclass instance to exact unicode - # raise a TypeError otherwise - if isinstance(s, str): - s = str.__str__(s) - else: - s = unicode.__getnewargs__(s)[0] - def replace(match): - return ESCAPE_DCT[match.group(0)] - return _q + ESCAPE.sub(replace, s) + _q - - -def py_encode_basestring_ascii(s, _PY3=PY3): - """Return an ASCII-only JSON representation of a Python string - - """ - if _PY3: - if isinstance(s, bytes): - s = str(s, 'utf-8') - elif type(s) is not str: - # convert an str subclass instance to exact str - # raise a TypeError otherwise - s = str.__str__(s) - else: - if isinstance(s, str) and HAS_UTF8.search(s) is not None: - s = unicode(s, 'utf-8') - elif type(s) not in (str, unicode): - # convert an str subclass instance to exact str - # convert a unicode subclass instance to exact unicode - # raise a TypeError otherwise - if isinstance(s, str): - s = str.__str__(s) - else: - s = unicode.__getnewargs__(s)[0] - def replace(match): - s = match.group(0) - try: - return ESCAPE_DCT[s] - except KeyError: - n = ord(s) - if n < 0x10000: - #return '\\u{0:04x}'.format(n) - return '\\u%04x' % (n,) - else: - # surrogate pair - n -= 0x10000 - s1 = 0xd800 | ((n >> 10) & 0x3ff) - s2 = 0xdc00 | (n & 0x3ff) - #return '\\u{0:04x}\\u{1:04x}'.format(s1, s2) - return '\\u%04x\\u%04x' % (s1, s2) - return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"' - - -encode_basestring_ascii = ( - c_encode_basestring_ascii or py_encode_basestring_ascii) - -class JSONEncoder(object): - """Extensible JSON <http://json.org> encoder for Python data structures. - - Supports the following objects and types by default: - - +-------------------+---------------+ - | Python | JSON | - +===================+===============+ - | dict, namedtuple | object | - +-------------------+---------------+ - | list, tuple | array | - +-------------------+---------------+ - | str, unicode | string | - +-------------------+---------------+ - | int, long, float | number | - +-------------------+---------------+ - | True | true | - +-------------------+---------------+ - | False | false | - +-------------------+---------------+ - | None | null | - +-------------------+---------------+ - - To extend this to recognize other objects, subclass and implement a - ``.default()`` method with another method that returns a serializable - object for ``o`` if possible, otherwise it should call the superclass - implementation (to raise ``TypeError``). - - """ - item_separator = ', ' - key_separator = ': ' - - def __init__(self, skipkeys=False, ensure_ascii=True, - check_circular=True, allow_nan=False, sort_keys=False, - indent=None, separators=None, encoding='utf-8', default=None, - use_decimal=True, namedtuple_as_object=True, - tuple_as_array=True, bigint_as_string=False, - item_sort_key=None, for_json=False, ignore_nan=False, - int_as_string_bitcount=None, iterable_as_array=False): - """Constructor for JSONEncoder, with sensible defaults. - - If skipkeys is false, then it is a TypeError to attempt - encoding of keys that are not str, int, long, float or None. If - skipkeys is True, such items are simply skipped. - - If ensure_ascii is true, the output is guaranteed to be str - objects with all incoming unicode characters escaped. If - ensure_ascii is false, the output will be unicode object. - - If check_circular is true, then lists, dicts, and custom encoded - objects will be checked for circular references during encoding to - prevent an infinite recursion (which would cause an OverflowError). - Otherwise, no such check takes place. - - If allow_nan is true (default: False), then out of range float - values (nan, inf, -inf) will be serialized to - their JavaScript equivalents (NaN, Infinity, -Infinity) - instead of raising a ValueError. See - ignore_nan for ECMA-262 compliant behavior. - - If sort_keys is true, then the output of dictionaries will be - sorted by key; this is useful for regression tests to ensure - that JSON serializations can be compared on a day-to-day basis. - - If indent is a string, then JSON array elements and object members - will be pretty-printed with a newline followed by that string repeated - for each level of nesting. ``None`` (the default) selects the most compact - representation without any newlines. For backwards compatibility with - versions of simplejson earlier than 2.1.0, an integer is also accepted - and is converted to a string with that many spaces. - - If specified, separators should be an (item_separator, key_separator) - tuple. The default is (', ', ': ') if *indent* is ``None`` and - (',', ': ') otherwise. To get the most compact JSON representation, - you should specify (',', ':') to eliminate whitespace. - - If specified, default is a function that gets called for objects - that can't otherwise be serialized. It should return a JSON encodable - version of the object or raise a ``TypeError``. - - If encoding is not None, then all input strings will be - transformed into unicode using that encoding prior to JSON-encoding. - The default is UTF-8. - - If use_decimal is true (default: ``True``), ``decimal.Decimal`` will - be supported directly by the encoder. For the inverse, decode JSON - with ``parse_float=decimal.Decimal``. - - If namedtuple_as_object is true (the default), objects with - ``_asdict()`` methods will be encoded as JSON objects. - - If tuple_as_array is true (the default), tuple (and subclasses) will - be encoded as JSON arrays. - - If *iterable_as_array* is true (default: ``False``), - any object not in the above table that implements ``__iter__()`` - will be encoded as a JSON array. - - If bigint_as_string is true (not the default), ints 2**53 and higher - or lower than -2**53 will be encoded as strings. This is to avoid the - rounding that happens in Javascript otherwise. - - If int_as_string_bitcount is a positive number (n), then int of size - greater than or equal to 2**n or lower than or equal to -2**n will be - encoded as strings. - - If specified, item_sort_key is a callable used to sort the items in - each dictionary. This is useful if you want to sort items other than - in alphabetical order by key. - - If for_json is true (not the default), objects with a ``for_json()`` - method will use the return value of that method for encoding as JSON - instead of the object. - - If *ignore_nan* is true (default: ``False``), then out of range - :class:`float` values (``nan``, ``inf``, ``-inf``) will be serialized - as ``null`` in compliance with the ECMA-262 specification. If true, - this will override *allow_nan*. - - """ - - self.skipkeys = skipkeys - self.ensure_ascii = ensure_ascii - self.check_circular = check_circular - self.allow_nan = allow_nan - self.sort_keys = sort_keys - self.use_decimal = use_decimal - self.namedtuple_as_object = namedtuple_as_object - self.tuple_as_array = tuple_as_array - self.iterable_as_array = iterable_as_array - self.bigint_as_string = bigint_as_string - self.item_sort_key = item_sort_key - self.for_json = for_json - self.ignore_nan = ignore_nan - self.int_as_string_bitcount = int_as_string_bitcount - if indent is not None and not isinstance(indent, string_types): - indent = indent * ' ' - self.indent = indent - if separators is not None: - self.item_separator, self.key_separator = separators - elif indent is not None: - self.item_separator = ',' - if default is not None: - self.default = default - self.encoding = encoding - - def default(self, o): - """Implement this method in a subclass such that it returns - a serializable object for ``o``, or calls the base implementation - (to raise a ``TypeError``). - - For example, to support arbitrary iterators, you could - implement default like this:: - - def default(self, o): - try: - iterable = iter(o) - except TypeError: - pass - else: - return list(iterable) - return JSONEncoder.default(self, o) - - """ - raise TypeError('Object of type %s is not JSON serializable' % - o.__class__.__name__) - - def encode(self, o): - """Return a JSON string representation of a Python data structure. - - >>> from simplejson import JSONEncoder - >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) - '{"foo": ["bar", "baz"]}' - - """ - # This is for extremely simple cases and benchmarks. - if isinstance(o, binary_type): - _encoding = self.encoding - if (_encoding is not None and not (_encoding == 'utf-8')): - o = text_type(o, _encoding) - if isinstance(o, string_types): - if self.ensure_ascii: - return encode_basestring_ascii(o) - else: - return encode_basestring(o) - # This doesn't pass the iterator directly to ''.join() because the - # exceptions aren't as detailed. The list call should be roughly - # equivalent to the PySequence_Fast that ''.join() would do. - chunks = self.iterencode(o) - if not isinstance(chunks, (list, tuple)): - chunks = list(chunks) - if self.ensure_ascii: - return ''.join(chunks) - else: - return u''.join(chunks) - - def iterencode(self, o): - """Encode the given object and yield each string - representation as available. - - For example:: - - for chunk in JSONEncoder().iterencode(bigobject): - mysocket.write(chunk) - - """ - if self.check_circular: - markers = {} - else: - markers = None - if self.ensure_ascii: - _encoder = encode_basestring_ascii - else: - _encoder = encode_basestring - if self.encoding != 'utf-8' and self.encoding is not None: - def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding): - if isinstance(o, binary_type): - o = text_type(o, _encoding) - return _orig_encoder(o) - - def floatstr(o, allow_nan=self.allow_nan, ignore_nan=self.ignore_nan, - _repr=FLOAT_REPR, _inf=PosInf, _neginf=-PosInf): - # Check for specials. Note that this type of test is processor - # and/or platform-specific, so do tests which don't depend on - # the internals. - - if o != o: - text = 'NaN' - elif o == _inf: - text = 'Infinity' - elif o == _neginf: - text = '-Infinity' - else: - if type(o) != float: - # See #118, do not trust custom str/repr - o = float(o) - return _repr(o) - - if ignore_nan: - text = 'null' - elif not allow_nan: - raise ValueError( - "Out of range float values are not JSON compliant: " + - repr(o)) - - return text - - key_memo = {} - int_as_string_bitcount = ( - 53 if self.bigint_as_string else self.int_as_string_bitcount) - if (c_make_encoder is not None and self.indent is None): - _iterencode = c_make_encoder( - markers, self.default, _encoder, self.indent, - self.key_separator, self.item_separator, self.sort_keys, - self.skipkeys, self.allow_nan, key_memo, self.use_decimal, - self.namedtuple_as_object, self.tuple_as_array, - int_as_string_bitcount, - self.item_sort_key, self.encoding, self.for_json, - self.ignore_nan, decimal.Decimal, self.iterable_as_array) - else: - _iterencode = _make_iterencode( - markers, self.default, _encoder, self.indent, floatstr, - self.key_separator, self.item_separator, self.sort_keys, - self.skipkeys, self.use_decimal, - self.namedtuple_as_object, self.tuple_as_array, - int_as_string_bitcount, - self.item_sort_key, self.encoding, self.for_json, - self.iterable_as_array, Decimal=decimal.Decimal) - try: - return _iterencode(o, 0) - finally: - key_memo.clear() - - -class JSONEncoderForHTML(JSONEncoder): - """An encoder that produces JSON safe to embed in HTML. - - To embed JSON content in, say, a script tag on a web page, the - characters &, < and > should be escaped. They cannot be escaped - with the usual entities (e.g. &) because they are not expanded - within <script> tags. - - This class also escapes the line separator and paragraph separator - characters U+2028 and U+2029, irrespective of the ensure_ascii setting, - as these characters are not valid in JavaScript strings (see - http://timelessrepo.com/json-isnt-a-javascript-subset). - """ - - def encode(self, o): - # Override JSONEncoder.encode because it has hacks for - # performance that make things more complicated. - chunks = self.iterencode(o) - if self.ensure_ascii: - return ''.join(chunks) - else: - return u''.join(chunks) - - def iterencode(self, o): - chunks = super(JSONEncoderForHTML, self).iterencode(o) - for chunk in chunks: - chunk = chunk.replace('&', '\\u0026') - chunk = chunk.replace('<', '\\u003c') - chunk = chunk.replace('>', '\\u003e') - - if not self.ensure_ascii: - chunk = chunk.replace(u'\u2028', '\\u2028') - chunk = chunk.replace(u'\u2029', '\\u2029') - - yield chunk - - -def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, - _key_separator, _item_separator, _sort_keys, _skipkeys, - _use_decimal, _namedtuple_as_object, _tuple_as_array, - _int_as_string_bitcount, _item_sort_key, - _encoding,_for_json, - _iterable_as_array, - ## HACK: hand-optimized bytecode; turn globals into locals - _PY3=PY3, - ValueError=ValueError, - string_types=string_types, - Decimal=None, - dict=dict, - float=float, - id=id, - integer_types=integer_types, - isinstance=isinstance, - list=list, - str=str, - tuple=tuple, - iter=iter, - ): - if _use_decimal and Decimal is None: - Decimal = decimal.Decimal - if _item_sort_key and not callable(_item_sort_key): - raise TypeError("item_sort_key must be None or callable") - elif _sort_keys and not _item_sort_key: - _item_sort_key = itemgetter(0) - - if (_int_as_string_bitcount is not None and - (_int_as_string_bitcount <= 0 or - not isinstance(_int_as_string_bitcount, integer_types))): - raise TypeError("int_as_string_bitcount must be a positive integer") - - def call_method(obj, method_name): - method = getattr(obj, method_name, None) - if callable(method): - try: - return (method(),) - except TypeError: - pass - return None - - def _encode_int(value): - skip_quoting = ( - _int_as_string_bitcount is None - or - _int_as_string_bitcount < 1 - ) - if type(value) not in integer_types: - # See #118, do not trust custom str/repr - value = int(value) - if ( - skip_quoting or - (-1 << _int_as_string_bitcount) - < value < - (1 << _int_as_string_bitcount) - ): - return str(value) - return '"' + str(value) + '"' - - def _iterencode_list(lst, _current_indent_level): - if not lst: - yield '[]' - return - if markers is not None: - markerid = id(lst) - if markerid in markers: - raise ValueError("Circular reference detected") - markers[markerid] = lst - buf = '[' - if _indent is not None: - _current_indent_level += 1 - newline_indent = '\n' + (_indent * _current_indent_level) - separator = _item_separator + newline_indent - buf += newline_indent - else: - newline_indent = None - separator = _item_separator - first = True - for value in lst: - if first: - first = False - else: - buf = separator - if isinstance(value, string_types): - yield buf + _encoder(value) - elif _PY3 and isinstance(value, bytes) and _encoding is not None: - yield buf + _encoder(value) - elif isinstance(value, RawJSON): - yield buf + value.encoded_json - elif value is None: - yield buf + 'null' - elif value is True: - yield buf + 'true' - elif value is False: - yield buf + 'false' - elif isinstance(value, integer_types): - yield buf + _encode_int(value) - elif isinstance(value, float): - yield buf + _floatstr(value) - elif _use_decimal and isinstance(value, Decimal): - yield buf + str(value) - else: - yield buf - for_json = _for_json and call_method(value, 'for_json') - if for_json: - chunks = _iterencode(for_json[0], _current_indent_level) - elif isinstance(value, list): - chunks = _iterencode_list(value, _current_indent_level) - else: - _asdict = _namedtuple_as_object and call_method(value, '_asdict') - if _asdict: - dct = _asdict[0] - if not isinstance(dct, dict): - raise TypeError("_asdict() must return a dict, not %s" % (type(dct).__name__,)) - chunks = _iterencode_dict(dct, - _current_indent_level) - elif _tuple_as_array and isinstance(value, tuple): - chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): - chunks = _iterencode_dict(value, _current_indent_level) - else: - chunks = _iterencode(value, _current_indent_level) - for chunk in chunks: - yield chunk - if first: - # iterable_as_array misses the fast path at the top - yield '[]' - else: - if newline_indent is not None: - _current_indent_level -= 1 - yield '\n' + (_indent * _current_indent_level) - yield ']' - if markers is not None: - del markers[markerid] - - def _stringify_key(key): - if isinstance(key, string_types): # pragma: no cover - pass - elif _PY3 and isinstance(key, bytes) and _encoding is not None: - key = str(key, _encoding) - elif isinstance(key, float): - key = _floatstr(key) - elif key is True: - key = 'true' - elif key is False: - key = 'false' - elif key is None: - key = 'null' - elif isinstance(key, integer_types): - if type(key) not in integer_types: - # See #118, do not trust custom str/repr - key = int(key) - key = str(key) - elif _use_decimal and isinstance(key, Decimal): - key = str(key) - elif _skipkeys: - key = None - else: - raise TypeError('keys must be str, int, float, bool or None, ' - 'not %s' % key.__class__.__name__) - return key - - def _iterencode_dict(dct, _current_indent_level): - if not dct: - yield '{}' - return - if markers is not None: - markerid = id(dct) - if markerid in markers: - raise ValueError("Circular reference detected") - markers[markerid] = dct - yield '{' - if _indent is not None: - _current_indent_level += 1 - newline_indent = '\n' + (_indent * _current_indent_level) - item_separator = _item_separator + newline_indent - yield newline_indent - else: - newline_indent = None - item_separator = _item_separator - first = True - if _PY3: - iteritems = dct.items() - else: - iteritems = dct.iteritems() - if _item_sort_key: - items = [] - for k, v in dct.items(): - if not isinstance(k, string_types): - k = _stringify_key(k) - if k is None: - continue - items.append((k, v)) - items.sort(key=_item_sort_key) - else: - items = iteritems - for key, value in items: - if not (_item_sort_key or isinstance(key, string_types)): - key = _stringify_key(key) - if key is None: - # _skipkeys must be True - continue - if first: - first = False - else: - yield item_separator - yield _encoder(key) - yield _key_separator - if isinstance(value, string_types): - yield _encoder(value) - elif _PY3 and isinstance(value, bytes) and _encoding is not None: - yield _encoder(value) - elif isinstance(value, RawJSON): - yield value.encoded_json - elif value is None: - yield 'null' - elif value is True: - yield 'true' - elif value is False: - yield 'false' - elif isinstance(value, integer_types): - yield _encode_int(value) - elif isinstance(value, float): - yield _floatstr(value) - elif _use_decimal and isinstance(value, Decimal): - yield str(value) - else: - for_json = _for_json and call_method(value, 'for_json') - if for_json: - chunks = _iterencode(for_json[0], _current_indent_level) - elif isinstance(value, list): - chunks = _iterencode_list(value, _current_indent_level) - else: - _asdict = _namedtuple_as_object and call_method(value, '_asdict') - if _asdict: - dct = _asdict[0] - if not isinstance(dct, dict): - raise TypeError("_asdict() must return a dict, not %s" % (type(dct).__name__,)) - chunks = _iterencode_dict(dct, - _current_indent_level) - elif _tuple_as_array and isinstance(value, tuple): - chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): - chunks = _iterencode_dict(value, _current_indent_level) - else: - chunks = _iterencode(value, _current_indent_level) - for chunk in chunks: - yield chunk - if newline_indent is not None: - _current_indent_level -= 1 - yield '\n' + (_indent * _current_indent_level) - yield '}' - if markers is not None: - del markers[markerid] - - def _iterencode(o, _current_indent_level): - if isinstance(o, string_types): - yield _encoder(o) - elif _PY3 and isinstance(o, bytes) and _encoding is not None: - yield _encoder(o) - elif isinstance(o, RawJSON): - yield o.encoded_json - elif o is None: - yield 'null' - elif o is True: - yield 'true' - elif o is False: - yield 'false' - elif isinstance(o, integer_types): - yield _encode_int(o) - elif isinstance(o, float): - yield _floatstr(o) - else: - for_json = _for_json and call_method(o, 'for_json') - if for_json: - for chunk in _iterencode(for_json[0], _current_indent_level): - yield chunk - elif isinstance(o, list): - for chunk in _iterencode_list(o, _current_indent_level): - yield chunk - else: - _asdict = _namedtuple_as_object and call_method(o, '_asdict') - if _asdict: - dct = _asdict[0] - if not isinstance(dct, dict): - raise TypeError("_asdict() must return a dict, not %s" % (type(dct).__name__,)) - for chunk in _iterencode_dict(dct, _current_indent_level): - yield chunk - elif (_tuple_as_array and isinstance(o, tuple)): - for chunk in _iterencode_list(o, _current_indent_level): - yield chunk - elif isinstance(o, dict): - for chunk in _iterencode_dict(o, _current_indent_level): - yield chunk - elif _use_decimal and isinstance(o, Decimal): - yield str(o) - else: - while _iterable_as_array: - # Markers are not checked here because it is valid for - # an iterable to return self. - try: - o = iter(o) - except TypeError: - break - for chunk in _iterencode_list(o, _current_indent_level): - yield chunk - return - if markers is not None: - markerid = id(o) - if markerid in markers: - raise ValueError("Circular reference detected") - markers[markerid] = o - o = _default(o) - for chunk in _iterencode(o, _current_indent_level): - yield chunk - if markers is not None: - del markers[markerid] - - return _iterencode diff --git a/contrib/python/simplejson/py2/simplejson/errors.py b/contrib/python/simplejson/py2/simplejson/errors.py deleted file mode 100644 index b97ab1e913f..00000000000 --- a/contrib/python/simplejson/py2/simplejson/errors.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Error classes used by simplejson -""" -__all__ = ['JSONDecodeError'] - - -def linecol(doc, pos): - lineno = doc.count('\n', 0, pos) + 1 - if lineno == 1: - colno = pos + 1 - else: - colno = pos - doc.rindex('\n', 0, pos) - return lineno, colno - - -def errmsg(msg, doc, pos, end=None): - lineno, colno = linecol(doc, pos) - msg = msg.replace('%r', repr(doc[pos:pos + 1])) - if end is None: - fmt = '%s: line %d column %d (char %d)' - return fmt % (msg, lineno, colno, pos) - endlineno, endcolno = linecol(doc, end) - fmt = '%s: line %d column %d - line %d column %d (char %d - %d)' - return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end) - - -class JSONDecodeError(ValueError): - """Subclass of ValueError with the following additional properties: - - msg: The unformatted error message - doc: The JSON document being parsed - pos: The start index of doc where parsing failed - end: The end index of doc where parsing failed (may be None) - lineno: The line corresponding to pos - colno: The column corresponding to pos - endlineno: The line corresponding to end (may be None) - endcolno: The column corresponding to end (may be None) - - """ - # Note that this exception is used from _speedups - def __init__(self, msg, doc, pos, end=None): - ValueError.__init__(self, errmsg(msg, doc, pos, end=end)) - self.msg = msg - self.doc = doc - self.pos = pos - self.end = end - self.lineno, self.colno = linecol(doc, pos) - if end is not None: - self.endlineno, self.endcolno = linecol(doc, end) - else: - self.endlineno, self.endcolno = None, None - - def __reduce__(self): - return self.__class__, (self.msg, self.doc, self.pos, self.end) diff --git a/contrib/python/simplejson/py2/simplejson/ordered_dict.py b/contrib/python/simplejson/py2/simplejson/ordered_dict.py deleted file mode 100644 index d5a55ebd0f7..00000000000 --- a/contrib/python/simplejson/py2/simplejson/ordered_dict.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Drop-in replacement for collections.OrderedDict by Raymond Hettinger - -http://code.activestate.com/recipes/576693/ - -""" -from UserDict import DictMixin - -class OrderedDict(dict, DictMixin): - - def __init__(self, *args, **kwds): - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__end - except AttributeError: - self.clear() - self.update(*args, **kwds) - - def clear(self): - self.__end = end = [] - end += [None, end, end] # sentinel node for doubly linked list - self.__map = {} # key --> [key, prev, next] - dict.clear(self) - - def __setitem__(self, key, value): - if key not in self: - end = self.__end - curr = end[1] - curr[2] = end[1] = self.__map[key] = [key, curr, end] - dict.__setitem__(self, key, value) - - def __delitem__(self, key): - dict.__delitem__(self, key) - key, prev, next = self.__map.pop(key) - prev[2] = next - next[1] = prev - - def __iter__(self): - end = self.__end - curr = end[2] - while curr is not end: - yield curr[0] - curr = curr[2] - - def __reversed__(self): - end = self.__end - curr = end[1] - while curr is not end: - yield curr[0] - curr = curr[1] - - def popitem(self, last=True): - if not self: - raise KeyError('dictionary is empty') - key = reversed(self).next() if last else iter(self).next() - value = self.pop(key) - return key, value - - def __reduce__(self): - items = [[k, self[k]] for k in self] - tmp = self.__map, self.__end - del self.__map, self.__end - inst_dict = vars(self).copy() - self.__map, self.__end = tmp - if inst_dict: - return (self.__class__, (items,), inst_dict) - return self.__class__, (items,) - - def keys(self): - return list(self) - - setdefault = DictMixin.setdefault - update = DictMixin.update - pop = DictMixin.pop - values = DictMixin.values - items = DictMixin.items - iterkeys = DictMixin.iterkeys - itervalues = DictMixin.itervalues - iteritems = DictMixin.iteritems - - def __repr__(self): - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, self.items()) - - def copy(self): - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - d = cls() - for key in iterable: - d[key] = value - return d - - def __eq__(self, other): - if isinstance(other, OrderedDict): - return len(self)==len(other) and \ - all(p==q for p, q in zip(self.items(), other.items())) - return dict.__eq__(self, other) - - def __ne__(self, other): - return not self == other diff --git a/contrib/python/simplejson/py2/simplejson/raw_json.py b/contrib/python/simplejson/py2/simplejson/raw_json.py deleted file mode 100644 index 2071a70206b..00000000000 --- a/contrib/python/simplejson/py2/simplejson/raw_json.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Implementation of RawJSON -""" - -class RawJSON(object): - """Wrap an encoded JSON document for direct embedding in the output - - """ - def __init__(self, encoded_json): - self.encoded_json = encoded_json diff --git a/contrib/python/simplejson/py2/simplejson/scanner.py b/contrib/python/simplejson/py2/simplejson/scanner.py deleted file mode 100644 index 34710d68c6d..00000000000 --- a/contrib/python/simplejson/py2/simplejson/scanner.py +++ /dev/null @@ -1,85 +0,0 @@ -"""JSON token scanner -""" -import re -from .errors import JSONDecodeError -def _import_c_make_scanner(): - try: - from ._speedups import make_scanner - return make_scanner - except ImportError: - return None -c_make_scanner = _import_c_make_scanner() - -__all__ = ['make_scanner', 'JSONDecodeError'] - -NUMBER_RE = re.compile( - r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?', - (re.VERBOSE | re.MULTILINE | re.DOTALL)) - - -def py_make_scanner(context): - parse_object = context.parse_object - parse_array = context.parse_array - parse_string = context.parse_string - match_number = NUMBER_RE.match - encoding = context.encoding - strict = context.strict - parse_float = context.parse_float - parse_int = context.parse_int - parse_constant = context.parse_constant - object_hook = context.object_hook - object_pairs_hook = context.object_pairs_hook - memo = context.memo - - def _scan_once(string, idx): - errmsg = 'Expecting value' - try: - nextchar = string[idx] - except IndexError: - raise JSONDecodeError(errmsg, string, idx) - - if nextchar == '"': - return parse_string(string, idx + 1, encoding, strict) - elif nextchar == '{': - return parse_object((string, idx + 1), encoding, strict, - _scan_once, object_hook, object_pairs_hook, memo) - elif nextchar == '[': - return parse_array((string, idx + 1), _scan_once) - elif nextchar == 'n' and string[idx:idx + 4] == 'null': - return None, idx + 4 - elif nextchar == 't' and string[idx:idx + 4] == 'true': - return True, idx + 4 - elif nextchar == 'f' and string[idx:idx + 5] == 'false': - return False, idx + 5 - - m = match_number(string, idx) - if m is not None: - integer, frac, exp = m.groups() - if frac or exp: - res = parse_float(integer + (frac or '') + (exp or '')) - else: - res = parse_int(integer) - return res, m.end() - elif parse_constant and nextchar == 'N' and string[idx:idx + 3] == 'NaN': - return parse_constant('NaN'), idx + 3 - elif parse_constant and nextchar == 'I' and string[idx:idx + 8] == 'Infinity': - return parse_constant('Infinity'), idx + 8 - elif parse_constant and nextchar == '-' and string[idx:idx + 9] == '-Infinity': - return parse_constant('-Infinity'), idx + 9 - else: - raise JSONDecodeError(errmsg, string, idx) - - def scan_once(string, idx): - if idx < 0: - # Ensure the same behavior as the C speedup, otherwise - # this would work for *some* negative string indices due - # to the behavior of __getitem__ for strings. #98 - raise JSONDecodeError('Expecting value', string, idx) - try: - return _scan_once(string, idx) - finally: - memo.clear() - - return scan_once - -make_scanner = c_make_scanner or py_make_scanner diff --git a/contrib/python/simplejson/py2/simplejson/tests/__init__.py b/contrib/python/simplejson/py2/simplejson/tests/__init__.py deleted file mode 100644 index d9986d033a0..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/__init__.py +++ /dev/null @@ -1,93 +0,0 @@ -from __future__ import absolute_import -import unittest -import sys -import os - - -class NoExtensionTestSuite(unittest.TestSuite): - def run(self, result): - import simplejson - - simplejson._toggle_speedups(False) - result = unittest.TestSuite.run(self, result) - simplejson._toggle_speedups(True) - return result - - -class TestMissingSpeedups(unittest.TestCase): - def runTest(self): - if hasattr(sys, "pypy_translation_info"): - "PyPy doesn't need speedups! :)" - elif getattr(getattr(sys, "implementation", None), "name", None) == "graalpy": - "GraalPy doesn't need speedups! :)" - elif hasattr(self, "skipTest"): - self.skipTest("_speedups.so is missing!") - - -def additional_tests(suite=None, project_dir=None): - import simplejson - import simplejson.encoder - import simplejson.decoder - - if suite is None: - suite = unittest.TestSuite() - try: - import doctest - except ImportError: - if sys.version_info < (2, 7): - # doctests in 2.6 depends on cStringIO - return suite - raise - for mod in (simplejson, simplejson.encoder, simplejson.decoder): - suite.addTest(doctest.DocTestSuite(mod)) - if project_dir is not None: - suite.addTest( - doctest.DocFileSuite( - os.path.join(project_dir, "index.rst"), module_relative=False - ) - ) - return suite - - -def all_tests_suite(project_dir=None): - def get_suite(): - suite_names = [ - "simplejson.tests.%s" % (os.path.splitext(f)[0],) - for f in os.listdir(os.path.dirname(__file__)) - if f.startswith("test_") and f.endswith(".py") - ] - return additional_tests( - suite=unittest.TestLoader().loadTestsFromNames(suite_names), - project_dir=project_dir, - ) - - suite = get_suite() - import simplejson - - if simplejson._import_c_make_encoder() is None: - suite.addTest(TestMissingSpeedups()) - else: - suite = unittest.TestSuite( - [ - suite, - NoExtensionTestSuite([get_suite()]), - ] - ) - return suite - - -def main(project_dir=None): - runner = unittest.TextTestRunner(verbosity=1 + sys.argv.count("-v")) - suite = all_tests_suite(project_dir=project_dir) - raise SystemExit(not runner.run(suite).wasSuccessful()) - - -if __name__ == "__main__": - import os - import sys - - project_dir = os.path.dirname( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - ) - sys.path.insert(0, project_dir) - main(project_dir=project_dir) diff --git a/contrib/python/simplejson/py2/simplejson/tests/_cibw_runner.py b/contrib/python/simplejson/py2/simplejson/tests/_cibw_runner.py deleted file mode 100644 index bffb70b8add..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/_cibw_runner.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Internal module for running tests from cibuildwheel""" - -import sys -import simplejson.tests - -if __name__ == '__main__': - simplejson.tests.main(project_dir=sys.argv[1]) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_bigint_as_string.py b/contrib/python/simplejson/py2/simplejson/tests/test_bigint_as_string.py deleted file mode 100644 index 2cf2cc2966e..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_bigint_as_string.py +++ /dev/null @@ -1,67 +0,0 @@ -from unittest import TestCase - -import simplejson as json - - -class TestBigintAsString(TestCase): - # Python 2.5, at least the one that ships on Mac OS X, calculates - # 2 ** 53 as 0! It manages to calculate 1 << 53 correctly. - values = [(200, 200), - ((1 << 53) - 1, 9007199254740991), - ((1 << 53), '9007199254740992'), - ((1 << 53) + 1, '9007199254740993'), - (-100, -100), - ((-1 << 53), '-9007199254740992'), - ((-1 << 53) - 1, '-9007199254740993'), - ((-1 << 53) + 1, -9007199254740991)] - - options = ( - {"bigint_as_string": True}, - {"int_as_string_bitcount": 53} - ) - - def test_ints(self): - for opts in self.options: - for val, expect in self.values: - self.assertEqual( - val, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, **opts))) - - def test_lists(self): - for opts in self.options: - for val, expect in self.values: - val = [val, val] - expect = [expect, expect] - self.assertEqual( - val, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, **opts))) - - def test_dicts(self): - for opts in self.options: - for val, expect in self.values: - val = {'k': val} - expect = {'k': expect} - self.assertEqual( - val, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, **opts))) - - def test_dict_keys(self): - for opts in self.options: - for val, _ in self.values: - expect = {str(val): 'value'} - val = {val: 'value'} - self.assertEqual( - expect, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, **opts))) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_bitsize_int_as_string.py b/contrib/python/simplejson/py2/simplejson/tests/test_bitsize_int_as_string.py deleted file mode 100644 index fd7d103a6d8..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_bitsize_int_as_string.py +++ /dev/null @@ -1,73 +0,0 @@ -from unittest import TestCase - -import simplejson as json - - -class TestBitSizeIntAsString(TestCase): - # Python 2.5, at least the one that ships on Mac OS X, calculates - # 2 ** 31 as 0! It manages to calculate 1 << 31 correctly. - values = [ - (200, 200), - ((1 << 31) - 1, (1 << 31) - 1), - ((1 << 31), str(1 << 31)), - ((1 << 31) + 1, str((1 << 31) + 1)), - (-100, -100), - ((-1 << 31), str(-1 << 31)), - ((-1 << 31) - 1, str((-1 << 31) - 1)), - ((-1 << 31) + 1, (-1 << 31) + 1), - ] - - def test_invalid_counts(self): - for n in ['foo', -1, 0, 1.0]: - self.assertRaises( - TypeError, - json.dumps, 0, int_as_string_bitcount=n) - - def test_ints_outside_range_fails(self): - self.assertNotEqual( - str(1 << 15), - json.loads(json.dumps(1 << 15, int_as_string_bitcount=16)), - ) - - def test_ints(self): - for val, expect in self.values: - self.assertEqual( - val, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, int_as_string_bitcount=31)), - ) - - def test_lists(self): - for val, expect in self.values: - val = [val, val] - expect = [expect, expect] - self.assertEqual( - val, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, int_as_string_bitcount=31))) - - def test_dicts(self): - for val, expect in self.values: - val = {'k': val} - expect = {'k': expect} - self.assertEqual( - val, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, int_as_string_bitcount=31))) - - def test_dict_keys(self): - for val, _ in self.values: - expect = {str(val): 'value'} - val = {val: 'value'} - self.assertEqual( - expect, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, int_as_string_bitcount=31))) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_check_circular.py b/contrib/python/simplejson/py2/simplejson/tests/test_check_circular.py deleted file mode 100644 index af6463d6d88..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_check_circular.py +++ /dev/null @@ -1,30 +0,0 @@ -from unittest import TestCase -import simplejson as json - -def default_iterable(obj): - return list(obj) - -class TestCheckCircular(TestCase): - def test_circular_dict(self): - dct = {} - dct['a'] = dct - self.assertRaises(ValueError, json.dumps, dct) - - def test_circular_list(self): - lst = [] - lst.append(lst) - self.assertRaises(ValueError, json.dumps, lst) - - def test_circular_composite(self): - dct2 = {} - dct2['a'] = [] - dct2['a'].append(dct2) - self.assertRaises(ValueError, json.dumps, dct2) - - def test_circular_default(self): - json.dumps([set()], default=default_iterable) - self.assertRaises(TypeError, json.dumps, [set()]) - - def test_circular_off_default(self): - json.dumps([set()], default=default_iterable, check_circular=False) - self.assertRaises(TypeError, json.dumps, [set()], check_circular=False) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_decimal.py b/contrib/python/simplejson/py2/simplejson/tests/test_decimal.py deleted file mode 100644 index 2b0940b1527..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_decimal.py +++ /dev/null @@ -1,71 +0,0 @@ -import decimal -from decimal import Decimal -from unittest import TestCase -from simplejson.compat import StringIO, reload_module - -import simplejson as json - -class TestDecimal(TestCase): - NUMS = "1.0", "10.00", "1.1", "1234567890.1234567890", "500" - def dumps(self, obj, **kw): - sio = StringIO() - json.dump(obj, sio, **kw) - res = json.dumps(obj, **kw) - self.assertEqual(res, sio.getvalue()) - return res - - def loads(self, s, **kw): - sio = StringIO(s) - res = json.loads(s, **kw) - self.assertEqual(res, json.load(sio, **kw)) - return res - - def test_decimal_encode(self): - for d in map(Decimal, self.NUMS): - self.assertEqual(self.dumps(d, use_decimal=True), str(d)) - - def test_decimal_decode(self): - for s in self.NUMS: - self.assertEqual(self.loads(s, parse_float=Decimal), Decimal(s)) - - def test_stringify_key(self): - for d in map(Decimal, self.NUMS): - v = {d: d} - self.assertEqual( - self.loads( - self.dumps(v, use_decimal=True), parse_float=Decimal), - {str(d): d}) - - def test_decimal_roundtrip(self): - for d in map(Decimal, self.NUMS): - # The type might not be the same (int and Decimal) but they - # should still compare equal. - for v in [d, [d], {'': d}]: - self.assertEqual( - self.loads( - self.dumps(v, use_decimal=True), parse_float=Decimal), - v) - - def test_decimal_defaults(self): - d = Decimal('1.1') - # use_decimal=True is the default - self.assertRaises(TypeError, json.dumps, d, use_decimal=False) - self.assertEqual('1.1', json.dumps(d)) - self.assertEqual('1.1', json.dumps(d, use_decimal=True)) - self.assertRaises(TypeError, json.dump, d, StringIO(), - use_decimal=False) - sio = StringIO() - json.dump(d, sio) - self.assertEqual('1.1', sio.getvalue()) - sio = StringIO() - json.dump(d, sio, use_decimal=True) - self.assertEqual('1.1', sio.getvalue()) - - def test_decimal_reload(self): - # Simulate a subinterpreter that reloads the Python modules but not - # the C code https://github.com/simplejson/simplejson/issues/34 - global Decimal - Decimal = reload_module(decimal).Decimal - import simplejson.encoder - simplejson.encoder.Decimal = Decimal - self.test_decimal_roundtrip() diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_decode.py b/contrib/python/simplejson/py2/simplejson/tests/test_decode.py deleted file mode 100644 index 317b4f98d55..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_decode.py +++ /dev/null @@ -1,127 +0,0 @@ -from __future__ import absolute_import -import decimal -from unittest import TestCase - -import sys -import simplejson as json -from simplejson.compat import StringIO, b, binary_type -from simplejson import OrderedDict - -class MisbehavingBytesSubtype(binary_type): - def decode(self, encoding=None): - return "bad decode" - def __str__(self): - return "bad __str__" - def __bytes__(self): - return b("bad __bytes__") - -class TestDecode(TestCase): - if not hasattr(TestCase, 'assertIs'): - def assertIs(self, a, b): - self.assertTrue(a is b, '%r is %r' % (a, b)) - - def test_decimal(self): - rval = json.loads('1.1', parse_float=decimal.Decimal) - self.assertTrue(isinstance(rval, decimal.Decimal)) - self.assertEqual(rval, decimal.Decimal('1.1')) - - def test_float(self): - rval = json.loads('1', parse_int=float) - self.assertTrue(isinstance(rval, float)) - self.assertEqual(rval, 1.0) - - def test_decoder_optimizations(self): - # Several optimizations were made that skip over calls to - # the whitespace regex, so this test is designed to try and - # exercise the uncommon cases. The array cases are already covered. - rval = json.loads('{ "key" : "value" , "k":"v" }') - self.assertEqual(rval, {"key":"value", "k":"v"}) - - def test_empty_objects(self): - s = '{}' - self.assertEqual(json.loads(s), eval(s)) - s = '[]' - self.assertEqual(json.loads(s), eval(s)) - s = '""' - self.assertEqual(json.loads(s), eval(s)) - - def test_object_pairs_hook(self): - s = '{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}' - p = [("xkd", 1), ("kcw", 2), ("art", 3), ("hxm", 4), - ("qrt", 5), ("pad", 6), ("hoy", 7)] - self.assertEqual(json.loads(s), eval(s)) - self.assertEqual(json.loads(s, object_pairs_hook=lambda x: x), p) - self.assertEqual(json.load(StringIO(s), - object_pairs_hook=lambda x: x), p) - od = json.loads(s, object_pairs_hook=OrderedDict) - self.assertEqual(od, OrderedDict(p)) - self.assertEqual(type(od), OrderedDict) - # the object_pairs_hook takes priority over the object_hook - self.assertEqual(json.loads(s, - object_pairs_hook=OrderedDict, - object_hook=lambda x: None), - OrderedDict(p)) - - def check_keys_reuse(self, source, loads): - rval = loads(source) - (a, b), (c, d) = sorted(rval[0]), sorted(rval[1]) - self.assertIs(a, c) - self.assertIs(b, d) - - def test_keys_reuse_str(self): - s = u'[{"a_key": 1, "b_\xe9": 2}, {"a_key": 3, "b_\xe9": 4}]'.encode('utf8') - self.check_keys_reuse(s, json.loads) - - def test_keys_reuse_unicode(self): - s = u'[{"a_key": 1, "b_\xe9": 2}, {"a_key": 3, "b_\xe9": 4}]' - self.check_keys_reuse(s, json.loads) - - def test_empty_strings(self): - self.assertEqual(json.loads('""'), "") - self.assertEqual(json.loads(u'""'), u"") - self.assertEqual(json.loads('[""]'), [""]) - self.assertEqual(json.loads(u'[""]'), [u""]) - - def test_raw_decode(self): - cls = json.decoder.JSONDecoder - self.assertEqual( - ({'a': {}}, 9), - cls().raw_decode("{\"a\": {}}")) - # http://code.google.com/p/simplejson/issues/detail?id=85 - self.assertEqual( - ({'a': {}}, 9), - cls(object_pairs_hook=dict).raw_decode("{\"a\": {}}")) - # https://github.com/simplejson/simplejson/pull/38 - self.assertEqual( - ({'a': {}}, 11), - cls().raw_decode(" \n{\"a\": {}}")) - - def test_bytes_decode(self): - cls = json.decoder.JSONDecoder - data = b('"\xe2\x82\xac"') - self.assertEqual(cls().decode(data), u'\u20ac') - self.assertEqual(cls(encoding='latin1').decode(data), u'\xe2\x82\xac') - self.assertEqual(cls(encoding=None).decode(data), u'\u20ac') - - data = MisbehavingBytesSubtype(b('"\xe2\x82\xac"')) - self.assertEqual(cls().decode(data), u'\u20ac') - self.assertEqual(cls(encoding='latin1').decode(data), u'\xe2\x82\xac') - self.assertEqual(cls(encoding=None).decode(data), u'\u20ac') - - def test_bounds_checking(self): - # https://github.com/simplejson/simplejson/issues/98 - j = json.decoder.JSONDecoder() - for i in [4, 5, 6, -1, -2, -3, -4, -5, -6]: - self.assertRaises(ValueError, j.scan_once, '1234', i) - self.assertRaises(ValueError, j.raw_decode, '1234', i) - x, y = sorted(['128931233', '472389423'], key=id) - diff = id(x) - id(y) - self.assertRaises(ValueError, j.scan_once, y, diff) - self.assertRaises(ValueError, j.raw_decode, y, i) - - def test_bounded_int(self): - # SJ-PT-23-03, limit quadratic number parsing per Python 3.11 - max_str_digits = getattr(sys, 'get_int_max_str_digits', lambda: 4300)() - s = '1' + '0' * (max_str_digits - 1) - self.assertEqual(json.loads(s), int(s)) - self.assertRaises(ValueError, json.loads, s + '0') diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_default.py b/contrib/python/simplejson/py2/simplejson/tests/test_default.py deleted file mode 100644 index d1eacb8c4c3..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_default.py +++ /dev/null @@ -1,9 +0,0 @@ -from unittest import TestCase - -import simplejson as json - -class TestDefault(TestCase): - def test_default(self): - self.assertEqual( - json.dumps(type, default=repr), - json.dumps(repr(type))) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_dump.py b/contrib/python/simplejson/py2/simplejson/tests/test_dump.py deleted file mode 100644 index 5628489f268..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_dump.py +++ /dev/null @@ -1,255 +0,0 @@ -from unittest import TestCase -from simplejson.compat import StringIO, long_type, b, binary_type, text_type, PY3 -import simplejson as json - -class MisbehavingTextSubtype(text_type): - def __str__(self): - return "FAIL!" - -class MisbehavingBytesSubtype(binary_type): - def decode(self, encoding=None): - return "bad decode" - def __str__(self): - return "bad __str__" - def __bytes__(self): - return b("bad __bytes__") - -def as_text_type(s): - if PY3 and isinstance(s, bytes): - return s.decode('ascii') - return s - -def decode_iso_8859_15(b): - return b.decode('iso-8859-15') - -class TestDump(TestCase): - def test_dump(self): - sio = StringIO() - json.dump({}, sio) - self.assertEqual(sio.getvalue(), '{}') - - def test_constants(self): - for c in [None, True, False]: - self.assertTrue(json.loads(json.dumps(c)) is c) - self.assertTrue(json.loads(json.dumps([c]))[0] is c) - self.assertTrue(json.loads(json.dumps({'a': c}))['a'] is c) - - def test_stringify_key(self): - items = [(b('bytes'), 'bytes'), - (1.0, '1.0'), - (10, '10'), - (True, 'true'), - (False, 'false'), - (None, 'null'), - (long_type(100), '100')] - for k, expect in items: - self.assertEqual( - json.loads(json.dumps({k: expect})), - {expect: expect}) - self.assertEqual( - json.loads(json.dumps({k: expect}, sort_keys=True)), - {expect: expect}) - self.assertRaises(TypeError, json.dumps, {json: 1}) - for v in [{}, {'other': 1}, {b('derp'): 1, 'herp': 2}]: - for sort_keys in [False, True]: - v0 = dict(v) - v0[json] = 1 - v1 = dict((as_text_type(key), val) for (key, val) in v.items()) - self.assertEqual( - json.loads(json.dumps(v0, skipkeys=True, sort_keys=sort_keys)), - v1) - self.assertEqual( - json.loads(json.dumps({'': v0}, skipkeys=True, sort_keys=sort_keys)), - {'': v1}) - self.assertEqual( - json.loads(json.dumps([v0], skipkeys=True, sort_keys=sort_keys)), - [v1]) - - def test_dumps(self): - self.assertEqual(json.dumps({}), '{}') - - def test_encode_truefalse(self): - self.assertEqual(json.dumps( - {True: False, False: True}, sort_keys=True), - '{"false": true, "true": false}') - self.assertEqual( - # load first because the keys are not sorted - json.loads(json.dumps({'k1': {False: 5}, 'k2': {0: 5}})), - {'k1': {'false': 5}, 'k2': {'0': 5}}, - ) - - self.assertEqual( - json.dumps( - {2: 3.0, - 4.0: long_type(5), - False: 1, - long_type(6): True, - "7": 0}, - sort_keys=True), - '{"2": 3.0, "4.0": 5, "6": true, "7": 0, "false": 1}') - - def test_ordered_dict(self): - # http://bugs.python.org/issue6105 - items = [('one', 1), ('two', 2), ('three', 3), ('four', 4), ('five', 5)] - s = json.dumps(json.OrderedDict(items)) - self.assertEqual( - s, - '{"one": 1, "two": 2, "three": 3, "four": 4, "five": 5}') - - def test_indent_unknown_type_acceptance(self): - """ - A test against the regression mentioned at `github issue 29`_. - - The indent parameter should accept any type which pretends to be - an instance of int or long when it comes to being multiplied by - strings, even if it is not actually an int or long, for - backwards compatibility. - - .. _github issue 29: - http://github.com/simplejson/simplejson/issue/29 - """ - - class AwesomeInt(object): - """An awesome reimplementation of integers""" - - def __init__(self, *args, **kwargs): - if len(args) > 0: - # [construct from literals, objects, etc.] - # ... - - # Finally, if args[0] is an integer, store it - if isinstance(args[0], int): - self._int = args[0] - - # [various methods] - - def __mul__(self, other): - # [various ways to multiply AwesomeInt objects] - # ... finally, if the right-hand operand is not awesome enough, - # try to do a normal integer multiplication - if hasattr(self, '_int'): - return self._int * other - else: - raise NotImplementedError("To do non-awesome things with" - " this object, please construct it from an integer!") - - s = json.dumps([0, 1, 2], indent=AwesomeInt(3)) - self.assertEqual(s, '[\n 0,\n 1,\n 2\n]') - - def test_accumulator(self): - # the C API uses an accumulator that collects after 100,000 appends - lst = [0] * 100000 - self.assertEqual(json.loads(json.dumps(lst)), lst) - - def test_sort_keys(self): - # https://github.com/simplejson/simplejson/issues/106 - for num_keys in range(2, 32): - p = dict((str(x), x) for x in range(num_keys)) - sio = StringIO() - json.dump(p, sio, sort_keys=True) - self.assertEqual(sio.getvalue(), json.dumps(p, sort_keys=True)) - self.assertEqual(json.loads(sio.getvalue()), p) - - def test_misbehaving_text_subtype(self): - # https://github.com/simplejson/simplejson/issues/185 - text = "this is some text" - self.assertEqual( - json.dumps(MisbehavingTextSubtype(text)), - json.dumps(text) - ) - self.assertEqual( - json.dumps([MisbehavingTextSubtype(text)]), - json.dumps([text]) - ) - self.assertEqual( - json.dumps({MisbehavingTextSubtype(text): 42}), - json.dumps({text: 42}) - ) - - def test_misbehaving_bytes_subtype(self): - data = b("this is some data \xe2\x82\xac") - self.assertEqual( - json.dumps(MisbehavingBytesSubtype(data)), - json.dumps(data) - ) - self.assertEqual( - json.dumps([MisbehavingBytesSubtype(data)]), - json.dumps([data]) - ) - self.assertEqual( - json.dumps({MisbehavingBytesSubtype(data): 42}), - json.dumps({data: 42}) - ) - - def test_bytes_toplevel(self): - self.assertEqual(json.dumps(b('\xe2\x82\xac')), r'"\u20ac"') - self.assertRaises(UnicodeDecodeError, json.dumps, b('\xa4')) - self.assertEqual(json.dumps(b('\xa4'), encoding='iso-8859-1'), - r'"\u00a4"') - self.assertEqual(json.dumps(b('\xa4'), encoding='iso-8859-15'), - r'"\u20ac"') - if PY3: - self.assertRaises(TypeError, json.dumps, b('\xe2\x82\xac'), - encoding=None) - self.assertRaises(TypeError, json.dumps, b('\xa4'), - encoding=None) - self.assertEqual(json.dumps(b('\xa4'), encoding=None, - default=decode_iso_8859_15), - r'"\u20ac"') - else: - self.assertEqual(json.dumps(b('\xe2\x82\xac'), encoding=None), - r'"\u20ac"') - self.assertRaises(UnicodeDecodeError, json.dumps, b('\xa4'), - encoding=None) - self.assertRaises(UnicodeDecodeError, json.dumps, b('\xa4'), - encoding=None, default=decode_iso_8859_15) - - def test_bytes_nested(self): - self.assertEqual(json.dumps([b('\xe2\x82\xac')]), r'["\u20ac"]') - self.assertRaises(UnicodeDecodeError, json.dumps, [b('\xa4')]) - self.assertEqual(json.dumps([b('\xa4')], encoding='iso-8859-1'), - r'["\u00a4"]') - self.assertEqual(json.dumps([b('\xa4')], encoding='iso-8859-15'), - r'["\u20ac"]') - if PY3: - self.assertRaises(TypeError, json.dumps, [b('\xe2\x82\xac')], - encoding=None) - self.assertRaises(TypeError, json.dumps, [b('\xa4')], - encoding=None) - self.assertEqual(json.dumps([b('\xa4')], encoding=None, - default=decode_iso_8859_15), - r'["\u20ac"]') - else: - self.assertEqual(json.dumps([b('\xe2\x82\xac')], encoding=None), - r'["\u20ac"]') - self.assertRaises(UnicodeDecodeError, json.dumps, [b('\xa4')], - encoding=None) - self.assertRaises(UnicodeDecodeError, json.dumps, [b('\xa4')], - encoding=None, default=decode_iso_8859_15) - - def test_bytes_key(self): - self.assertEqual(json.dumps({b('\xe2\x82\xac'): 42}), r'{"\u20ac": 42}') - self.assertRaises(UnicodeDecodeError, json.dumps, {b('\xa4'): 42}) - self.assertEqual(json.dumps({b('\xa4'): 42}, encoding='iso-8859-1'), - r'{"\u00a4": 42}') - self.assertEqual(json.dumps({b('\xa4'): 42}, encoding='iso-8859-15'), - r'{"\u20ac": 42}') - if PY3: - self.assertRaises(TypeError, json.dumps, {b('\xe2\x82\xac'): 42}, - encoding=None) - self.assertRaises(TypeError, json.dumps, {b('\xa4'): 42}, - encoding=None) - self.assertRaises(TypeError, json.dumps, {b('\xa4'): 42}, - encoding=None, default=decode_iso_8859_15) - self.assertEqual(json.dumps({b('\xa4'): 42}, encoding=None, - skipkeys=True), - r'{}') - else: - self.assertEqual(json.dumps({b('\xe2\x82\xac'): 42}, encoding=None), - r'{"\u20ac": 42}') - self.assertRaises(UnicodeDecodeError, json.dumps, {b('\xa4'): 42}, - encoding=None) - self.assertRaises(UnicodeDecodeError, json.dumps, {b('\xa4'): 42}, - encoding=None, default=decode_iso_8859_15) - self.assertRaises(UnicodeDecodeError, json.dumps, {b('\xa4'): 42}, - encoding=None, skipkeys=True) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_encode_basestring_ascii.py b/contrib/python/simplejson/py2/simplejson/tests/test_encode_basestring_ascii.py deleted file mode 100644 index 49706bfd978..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_encode_basestring_ascii.py +++ /dev/null @@ -1,47 +0,0 @@ -from unittest import TestCase - -import simplejson.encoder -from simplejson.compat import b - -CASES = [ - (u'/\\"\ucafe\ubabe\uab98\ufcde\ubcda\uef4a\x08\x0c\n\r\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?', '"/\\\\\\"\\ucafe\\ubabe\\uab98\\ufcde\\ubcda\\uef4a\\b\\f\\n\\r\\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?"'), - (u'\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), - (u'controls', '"controls"'), - (u'\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'), - (u'{"object with 1 member":["array with 1 element"]}', '"{\\"object with 1 member\\":[\\"array with 1 element\\"]}"'), - (u' s p a c e d ', '" s p a c e d "'), - (u'\U0001d120', '"\\ud834\\udd20"'), - (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'), - (b('\xce\xb1\xce\xa9'), '"\\u03b1\\u03a9"'), - (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'), - (b('\xce\xb1\xce\xa9'), '"\\u03b1\\u03a9"'), - (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'), - (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'), - (u"`1~!@#$%^&*()_+-={':[,]}|;.</>?", '"`1~!@#$%^&*()_+-={\':[,]}|;.</>?"'), - (u'\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'), - (u'\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), -] - -class TestEncodeBaseStringAscii(TestCase): - def test_py_encode_basestring_ascii(self): - self._test_encode_basestring_ascii(simplejson.encoder.py_encode_basestring_ascii) - - def test_c_encode_basestring_ascii(self): - if not simplejson.encoder.c_encode_basestring_ascii: - return - self._test_encode_basestring_ascii(simplejson.encoder.c_encode_basestring_ascii) - - def _test_encode_basestring_ascii(self, encode_basestring_ascii): - fname = encode_basestring_ascii.__name__ - for input_string, expect in CASES: - result = encode_basestring_ascii(input_string) - #self.assertEqual(result, expect, - # '{0!r} != {1!r} for {2}({3!r})'.format( - # result, expect, fname, input_string)) - self.assertEqual(result, expect, - '%r != %r for %s(%r)' % (result, expect, fname, input_string)) - - def test_sorted_dict(self): - items = [('one', 1), ('two', 2), ('three', 3), ('four', 4), ('five', 5)] - s = simplejson.dumps(dict(items), sort_keys=True) - self.assertEqual(s, '{"five": 5, "four": 4, "one": 1, "three": 3, "two": 2}') diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_encode_for_html.py b/contrib/python/simplejson/py2/simplejson/tests/test_encode_for_html.py deleted file mode 100644 index 3a840aa48cb..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_encode_for_html.py +++ /dev/null @@ -1,38 +0,0 @@ -import unittest - -import simplejson as json - -class TestEncodeForHTML(unittest.TestCase): - - def setUp(self): - self.decoder = json.JSONDecoder() - self.encoder = json.JSONEncoderForHTML() - self.non_ascii_encoder = json.JSONEncoderForHTML(ensure_ascii=False) - - def test_basic_encode(self): - self.assertEqual(r'"\u0026"', self.encoder.encode('&')) - self.assertEqual(r'"\u003c"', self.encoder.encode('<')) - self.assertEqual(r'"\u003e"', self.encoder.encode('>')) - self.assertEqual(r'"\u2028"', self.encoder.encode(u'\u2028')) - - def test_non_ascii_basic_encode(self): - self.assertEqual(r'"\u0026"', self.non_ascii_encoder.encode('&')) - self.assertEqual(r'"\u003c"', self.non_ascii_encoder.encode('<')) - self.assertEqual(r'"\u003e"', self.non_ascii_encoder.encode('>')) - self.assertEqual(r'"\u2028"', self.non_ascii_encoder.encode(u'\u2028')) - - def test_basic_roundtrip(self): - for char in '&<>': - self.assertEqual( - char, self.decoder.decode( - self.encoder.encode(char))) - - def test_prevent_script_breakout(self): - bad_string = '</script><script>alert("gotcha")</script>' - self.assertEqual( - r'"\u003c/script\u003e\u003cscript\u003e' - r'alert(\"gotcha\")\u003c/script\u003e"', - self.encoder.encode(bad_string)) - self.assertEqual( - bad_string, self.decoder.decode( - self.encoder.encode(bad_string))) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_errors.py b/contrib/python/simplejson/py2/simplejson/tests/test_errors.py deleted file mode 100644 index d5738254541..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_errors.py +++ /dev/null @@ -1,68 +0,0 @@ -import sys, pickle -from unittest import TestCase - -import simplejson as json -from simplejson.compat import text_type, b - -class TestErrors(TestCase): - def test_string_keys_error(self): - data = [{'a': 'A', 'b': (2, 4), 'c': 3.0, ('d',): 'D tuple'}] - try: - json.dumps(data) - except TypeError: - err = sys.exc_info()[1] - else: - self.fail('Expected TypeError') - self.assertEqual(str(err), - 'keys must be str, int, float, bool or None, not tuple') - - def test_not_serializable(self): - try: - json.dumps(json) - except TypeError: - err = sys.exc_info()[1] - else: - self.fail('Expected TypeError') - self.assertEqual(str(err), - 'Object of type module is not JSON serializable') - - def test_decode_error(self): - err = None - try: - json.loads('{}\na\nb') - except json.JSONDecodeError: - err = sys.exc_info()[1] - else: - self.fail('Expected JSONDecodeError') - self.assertEqual(err.lineno, 2) - self.assertEqual(err.colno, 1) - self.assertEqual(err.endlineno, 3) - self.assertEqual(err.endcolno, 2) - - def test_scan_error(self): - err = None - for t in (text_type, b): - try: - json.loads(t('{"asdf": "')) - except json.JSONDecodeError: - err = sys.exc_info()[1] - else: - self.fail('Expected JSONDecodeError') - self.assertEqual(err.lineno, 1) - self.assertEqual(err.colno, 10) - - def test_error_is_pickable(self): - err = None - try: - json.loads('{}\na\nb') - except json.JSONDecodeError: - err = sys.exc_info()[1] - else: - self.fail('Expected JSONDecodeError') - s = pickle.dumps(err) - e = pickle.loads(s) - - self.assertEqual(err.msg, e.msg) - self.assertEqual(err.doc, e.doc) - self.assertEqual(err.pos, e.pos) - self.assertEqual(err.end, e.end) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_fail.py b/contrib/python/simplejson/py2/simplejson/tests/test_fail.py deleted file mode 100644 index 5f9a8f698cf..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_fail.py +++ /dev/null @@ -1,178 +0,0 @@ -import sys -from unittest import TestCase - -import simplejson as json - -# 2007-10-05 -JSONDOCS = [ - # http://json.org/JSON_checker/test/fail1.json - '"A JSON payload should be an object or array, not a string."', - # http://json.org/JSON_checker/test/fail2.json - '["Unclosed array"', - # http://json.org/JSON_checker/test/fail3.json - '{unquoted_key: "keys must be quoted"}', - # http://json.org/JSON_checker/test/fail4.json - '["extra comma",]', - # http://json.org/JSON_checker/test/fail5.json - '["double extra comma",,]', - # http://json.org/JSON_checker/test/fail6.json - '[ , "<-- missing value"]', - # http://json.org/JSON_checker/test/fail7.json - '["Comma after the close"],', - # http://json.org/JSON_checker/test/fail8.json - '["Extra close"]]', - # http://json.org/JSON_checker/test/fail9.json - '{"Extra comma": true,}', - # http://json.org/JSON_checker/test/fail10.json - '{"Extra value after close": true} "misplaced quoted value"', - # http://json.org/JSON_checker/test/fail11.json - '{"Illegal expression": 1 + 2}', - # http://json.org/JSON_checker/test/fail12.json - '{"Illegal invocation": alert()}', - # http://json.org/JSON_checker/test/fail13.json - '{"Numbers cannot have leading zeroes": 013}', - # http://json.org/JSON_checker/test/fail14.json - '{"Numbers cannot be hex": 0x14}', - # http://json.org/JSON_checker/test/fail15.json - '["Illegal backslash escape: \\x15"]', - # http://json.org/JSON_checker/test/fail16.json - '[\\naked]', - # http://json.org/JSON_checker/test/fail17.json - '["Illegal backslash escape: \\017"]', - # http://json.org/JSON_checker/test/fail18.json - '[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]', - # http://json.org/JSON_checker/test/fail19.json - '{"Missing colon" null}', - # http://json.org/JSON_checker/test/fail20.json - '{"Double colon":: null}', - # http://json.org/JSON_checker/test/fail21.json - '{"Comma instead of colon", null}', - # http://json.org/JSON_checker/test/fail22.json - '["Colon instead of comma": false]', - # http://json.org/JSON_checker/test/fail23.json - '["Bad value", truth]', - # http://json.org/JSON_checker/test/fail24.json - "['single quote']", - # http://json.org/JSON_checker/test/fail25.json - '["\ttab\tcharacter\tin\tstring\t"]', - # http://json.org/JSON_checker/test/fail26.json - '["tab\\ character\\ in\\ string\\ "]', - # http://json.org/JSON_checker/test/fail27.json - '["line\nbreak"]', - # http://json.org/JSON_checker/test/fail28.json - '["line\\\nbreak"]', - # http://json.org/JSON_checker/test/fail29.json - '[0e]', - # http://json.org/JSON_checker/test/fail30.json - '[0e+]', - # http://json.org/JSON_checker/test/fail31.json - '[0e+-1]', - # http://json.org/JSON_checker/test/fail32.json - '{"Comma instead if closing brace": true,', - # http://json.org/JSON_checker/test/fail33.json - '["mismatch"}', - # http://code.google.com/p/simplejson/issues/detail?id=3 - u'["A\u001FZ control characters in string"]', - # misc based on coverage - '{', - '{]', - '{"foo": "bar"]', - '{"foo": "bar"', - 'nul', - 'nulx', - '-', - '-x', - '-e', - '-e0', - '-Infinite', - '-Inf', - 'Infinit', - 'Infinite', - 'NaM', - 'NuN', - 'falsy', - 'fal', - 'trug', - 'tru', - '1e', - '1ex', - '1e-', - '1e-x', -] - -SKIPS = { - 1: "why not have a string payload?", - 18: "spec doesn't specify any nesting limitations", -} - -class TestFail(TestCase): - def test_failures(self): - for idx, doc in enumerate(JSONDOCS): - idx = idx + 1 - if idx in SKIPS: - json.loads(doc) - continue - try: - json.loads(doc) - except json.JSONDecodeError: - pass - else: - self.fail("Expected failure for fail%d.json: %r" % (idx, doc)) - - def test_array_decoder_issue46(self): - # http://code.google.com/p/simplejson/issues/detail?id=46 - for doc in [u'[,]', '[,]']: - try: - json.loads(doc) - except json.JSONDecodeError: - e = sys.exc_info()[1] - self.assertEqual(e.pos, 1) - self.assertEqual(e.lineno, 1) - self.assertEqual(e.colno, 2) - except Exception: - e = sys.exc_info()[1] - self.fail("Unexpected exception raised %r %s" % (e, e)) - else: - self.fail("Unexpected success parsing '[,]'") - - def test_truncated_input(self): - test_cases = [ - ('', 'Expecting value', 0), - ('[', "Expecting value or ']'", 1), - ('[42', "Expecting ',' delimiter", 3), - ('[42,', 'Expecting value', 4), - ('["', 'Unterminated string starting at', 1), - ('["spam', 'Unterminated string starting at', 1), - ('["spam"', "Expecting ',' delimiter", 7), - ('["spam",', 'Expecting value', 8), - ('{', "Expecting property name enclosed in double quotes or '}'", 1), - ('{"', 'Unterminated string starting at', 1), - ('{"spam', 'Unterminated string starting at', 1), - ('{"spam"', "Expecting ':' delimiter", 7), - ('{"spam":', 'Expecting value', 8), - ('{"spam":42', "Expecting ',' delimiter", 10), - ('{"spam":42,', 'Expecting property name enclosed in double quotes', - 11), - ('"', 'Unterminated string starting at', 0), - ('"spam', 'Unterminated string starting at', 0), - ('[,', "Expecting value", 1), - ('--', 'Expecting value', 0), - ('"\x18d', "Invalid control character %r", 1), - ] - for data, msg, idx in test_cases: - try: - json.loads(data) - except json.JSONDecodeError: - e = sys.exc_info()[1] - self.assertEqual( - e.msg[:len(msg)], - msg, - "%r doesn't start with %r for %r" % (e.msg, msg, data)) - self.assertEqual( - e.pos, idx, - "pos %r != %r for %r" % (e.pos, idx, data)) - except Exception: - e = sys.exc_info()[1] - self.fail("Unexpected exception raised %r %s" % (e, e)) - else: - self.fail("Unexpected success parsing '%r'" % (data,)) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_float.py b/contrib/python/simplejson/py2/simplejson/tests/test_float.py deleted file mode 100644 index fed1bd97cd6..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_float.py +++ /dev/null @@ -1,46 +0,0 @@ -import sys -import math -from unittest import TestCase -from simplejson.compat import long_type, text_type -import simplejson as json -from simplejson.decoder import NaN, PosInf, NegInf - -class TestFloat(TestCase): - def test_degenerates_allow(self): - for inf in (PosInf, NegInf): - self.assertEqual(json.loads(json.dumps(inf, allow_nan=True), allow_nan=True), inf) - # Python 2.5 doesn't have math.isnan - nan = json.loads(json.dumps(NaN, allow_nan=True), allow_nan=True) - self.assertTrue((0 + nan) != nan) - - def test_degenerates_ignore(self): - for f in (PosInf, NegInf, NaN): - self.assertEqual(json.loads(json.dumps(f, ignore_nan=True)), None) - - def test_degenerates_deny(self): - for f in (PosInf, NegInf, NaN): - self.assertRaises(ValueError, json.dumps, f, allow_nan=False) - for s in ('Infinity', '-Infinity', 'NaN'): - self.assertRaises(ValueError, json.loads, s, allow_nan=False) - self.assertRaises(ValueError, json.loads, s) - - def test_floats(self): - for num in [1617161771.7650001, math.pi, math.pi**100, - math.pi**-100, 3.1]: - self.assertEqual(float(json.dumps(num)), num) - self.assertEqual(json.loads(json.dumps(num)), num) - self.assertEqual(json.loads(text_type(json.dumps(num))), num) - - def test_ints(self): - for num in [1, long_type(1), 1<<32, 1<<64]: - self.assertEqual(json.dumps(num), str(num)) - self.assertEqual(int(json.dumps(num)), num) - self.assertEqual(json.loads(json.dumps(num)), num) - self.assertEqual(json.loads(text_type(json.dumps(num))), num) - - def test_float_range(self): - try: - float_range = [sys.float_info.min, sys.float_info.max] - except AttributeError: - float_range = [2.2250738585072014e-308, 1.7976931348623157e+308] - self.assertEqual(json.loads(json.dumps(float_range)), float_range) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_for_json.py b/contrib/python/simplejson/py2/simplejson/tests/test_for_json.py deleted file mode 100644 index b791b883b04..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_for_json.py +++ /dev/null @@ -1,97 +0,0 @@ -import unittest -import simplejson as json - - -class ForJson(object): - def for_json(self): - return {'for_json': 1} - - -class NestedForJson(object): - def for_json(self): - return {'nested': ForJson()} - - -class ForJsonList(object): - def for_json(self): - return ['list'] - - -class DictForJson(dict): - def for_json(self): - return {'alpha': 1} - - -class ListForJson(list): - def for_json(self): - return ['list'] - - -class TestForJson(unittest.TestCase): - def assertRoundTrip(self, obj, other, for_json=True): - if for_json is None: - # None will use the default - s = json.dumps(obj) - else: - s = json.dumps(obj, for_json=for_json) - self.assertEqual( - json.loads(s), - other) - - def test_for_json_encodes_stand_alone_object(self): - self.assertRoundTrip( - ForJson(), - ForJson().for_json()) - - def test_for_json_encodes_object_nested_in_dict(self): - self.assertRoundTrip( - {'hooray': ForJson()}, - {'hooray': ForJson().for_json()}) - - def test_for_json_encodes_object_nested_in_list_within_dict(self): - self.assertRoundTrip( - {'list': [0, ForJson(), 2, 3]}, - {'list': [0, ForJson().for_json(), 2, 3]}) - - def test_for_json_encodes_object_nested_within_object(self): - self.assertRoundTrip( - NestedForJson(), - {'nested': {'for_json': 1}}) - - def test_for_json_encodes_list(self): - self.assertRoundTrip( - ForJsonList(), - ForJsonList().for_json()) - - def test_for_json_encodes_list_within_object(self): - self.assertRoundTrip( - {'nested': ForJsonList()}, - {'nested': ForJsonList().for_json()}) - - def test_for_json_encodes_dict_subclass(self): - self.assertRoundTrip( - DictForJson(a=1), - DictForJson(a=1).for_json()) - - def test_for_json_encodes_list_subclass(self): - self.assertRoundTrip( - ListForJson(['l']), - ListForJson(['l']).for_json()) - - def test_for_json_ignored_if_not_true_with_dict_subclass(self): - for for_json in (None, False): - self.assertRoundTrip( - DictForJson(a=1), - {'a': 1}, - for_json=for_json) - - def test_for_json_ignored_if_not_true_with_list_subclass(self): - for for_json in (None, False): - self.assertRoundTrip( - ListForJson(['l']), - ['l'], - for_json=for_json) - - def test_raises_typeerror_if_for_json_not_true_with_object(self): - self.assertRaises(TypeError, json.dumps, ForJson()) - self.assertRaises(TypeError, json.dumps, ForJson(), for_json=False) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_indent.py b/contrib/python/simplejson/py2/simplejson/tests/test_indent.py deleted file mode 100644 index cea25a575ed..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_indent.py +++ /dev/null @@ -1,86 +0,0 @@ -from unittest import TestCase -import textwrap - -import simplejson as json -from simplejson.compat import StringIO - -class TestIndent(TestCase): - def test_indent(self): - h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', - 'i-vhbjkhnth', - {'nifty': 87}, {'field': 'yes', 'morefield': False} ] - - expect = textwrap.dedent("""\ - [ - \t[ - \t\t"blorpie" - \t], - \t[ - \t\t"whoops" - \t], - \t[], - \t"d-shtaeou", - \t"d-nthiouh", - \t"i-vhbjkhnth", - \t{ - \t\t"nifty": 87 - \t}, - \t{ - \t\t"field": "yes", - \t\t"morefield": false - \t} - ]""") - - - d1 = json.dumps(h) - d2 = json.dumps(h, indent='\t', sort_keys=True, separators=(',', ': ')) - d3 = json.dumps(h, indent=' ', sort_keys=True, separators=(',', ': ')) - d4 = json.dumps(h, indent=2, sort_keys=True, separators=(',', ': ')) - - h1 = json.loads(d1) - h2 = json.loads(d2) - h3 = json.loads(d3) - h4 = json.loads(d4) - - self.assertEqual(h1, h) - self.assertEqual(h2, h) - self.assertEqual(h3, h) - self.assertEqual(h4, h) - self.assertEqual(d3, expect.replace('\t', ' ')) - self.assertEqual(d4, expect.replace('\t', ' ')) - # NOTE: Python 2.4 textwrap.dedent converts tabs to spaces, - # so the following is expected to fail. Python 2.4 is not a - # supported platform in simplejson 2.1.0+. - self.assertEqual(d2, expect) - - def test_indent0(self): - h = {3: 1} - def check(indent, expected): - d1 = json.dumps(h, indent=indent) - self.assertEqual(d1, expected) - - sio = StringIO() - json.dump(h, sio, indent=indent) - self.assertEqual(sio.getvalue(), expected) - - # indent=0 should emit newlines - check(0, '{\n"3": 1\n}') - # indent=None is more compact - check(None, '{"3": 1}') - - def test_separators(self): - lst = [1,2,3,4] - expect = '[\n1,\n2,\n3,\n4\n]' - expect_spaces = '[\n1, \n2, \n3, \n4\n]' - # Ensure that separators still works - self.assertEqual( - expect_spaces, - json.dumps(lst, indent=0, separators=(', ', ': '))) - # Force the new defaults - self.assertEqual( - expect, - json.dumps(lst, indent=0, separators=(',', ': '))) - # Added in 2.1.4 - self.assertEqual( - expect, - json.dumps(lst, indent=0)) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_item_sort_key.py b/contrib/python/simplejson/py2/simplejson/tests/test_item_sort_key.py deleted file mode 100644 index 98971b86ac7..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_item_sort_key.py +++ /dev/null @@ -1,27 +0,0 @@ -from unittest import TestCase - -import simplejson as json -from operator import itemgetter - -class TestItemSortKey(TestCase): - def test_simple_first(self): - a = {'a': 1, 'c': 5, 'jack': 'jill', 'pick': 'axe', 'array': [1, 5, 6, 9], 'tuple': (83, 12, 3), 'crate': 'dog', 'zeak': 'oh'} - self.assertEqual( - '{"a": 1, "c": 5, "crate": "dog", "jack": "jill", "pick": "axe", "zeak": "oh", "array": [1, 5, 6, 9], "tuple": [83, 12, 3]}', - json.dumps(a, item_sort_key=json.simple_first)) - - def test_case(self): - a = {'a': 1, 'c': 5, 'Jack': 'jill', 'pick': 'axe', 'Array': [1, 5, 6, 9], 'tuple': (83, 12, 3), 'crate': 'dog', 'zeak': 'oh'} - self.assertEqual( - '{"Array": [1, 5, 6, 9], "Jack": "jill", "a": 1, "c": 5, "crate": "dog", "pick": "axe", "tuple": [83, 12, 3], "zeak": "oh"}', - json.dumps(a, item_sort_key=itemgetter(0))) - self.assertEqual( - '{"a": 1, "Array": [1, 5, 6, 9], "c": 5, "crate": "dog", "Jack": "jill", "pick": "axe", "tuple": [83, 12, 3], "zeak": "oh"}', - json.dumps(a, item_sort_key=lambda kv: kv[0].lower())) - - def test_item_sort_key_value(self): - # https://github.com/simplejson/simplejson/issues/173 - a = {'a': 1, 'b': 0} - self.assertEqual( - '{"b": 0, "a": 1}', - json.dumps(a, item_sort_key=lambda kv: kv[1])) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_iterable.py b/contrib/python/simplejson/py2/simplejson/tests/test_iterable.py deleted file mode 100644 index 35d3e759111..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_iterable.py +++ /dev/null @@ -1,31 +0,0 @@ -import unittest -from simplejson.compat import StringIO - -import simplejson as json - -def iter_dumps(obj, **kw): - return ''.join(json.JSONEncoder(**kw).iterencode(obj)) - -def sio_dump(obj, **kw): - sio = StringIO() - json.dumps(obj, **kw) - return sio.getvalue() - -class TestIterable(unittest.TestCase): - def test_iterable(self): - for l in ([], [1], [1, 2], [1, 2, 3]): - for opts in [{}, {'indent': 2}]: - for dumps in (json.dumps, iter_dumps, sio_dump): - expect = dumps(l, **opts) - default_expect = dumps(sum(l), **opts) - # Default is False - self.assertRaises(TypeError, dumps, iter(l), **opts) - self.assertRaises(TypeError, dumps, iter(l), iterable_as_array=False, **opts) - self.assertEqual(expect, dumps(iter(l), iterable_as_array=True, **opts)) - # Ensure that the "default" gets called - self.assertEqual(default_expect, dumps(iter(l), default=sum, **opts)) - self.assertEqual(default_expect, dumps(iter(l), iterable_as_array=False, default=sum, **opts)) - # Ensure that the "default" does not get called - self.assertEqual( - expect, - dumps(iter(l), iterable_as_array=True, default=sum, **opts)) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_namedtuple.py b/contrib/python/simplejson/py2/simplejson/tests/test_namedtuple.py deleted file mode 100644 index cc0f8aa1d7d..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_namedtuple.py +++ /dev/null @@ -1,174 +0,0 @@ -from __future__ import absolute_import -import unittest -import simplejson as json -from simplejson.compat import StringIO - -try: - from unittest import mock -except ImportError: - mock = None - -try: - from collections import namedtuple -except ImportError: - class Value(tuple): - def __new__(cls, *args): - return tuple.__new__(cls, args) - - def _asdict(self): - return {'value': self[0]} - class Point(tuple): - def __new__(cls, *args): - return tuple.__new__(cls, args) - - def _asdict(self): - return {'x': self[0], 'y': self[1]} -else: - Value = namedtuple('Value', ['value']) - Point = namedtuple('Point', ['x', 'y']) - -class DuckValue(object): - def __init__(self, *args): - self.value = Value(*args) - - def _asdict(self): - return self.value._asdict() - -class DuckPoint(object): - def __init__(self, *args): - self.point = Point(*args) - - def _asdict(self): - return self.point._asdict() - -class DeadDuck(object): - _asdict = None - -class DeadDict(dict): - _asdict = None - -CONSTRUCTORS = [ - lambda v: v, - lambda v: [v], - lambda v: [{'key': v}], -] - -class TestNamedTuple(unittest.TestCase): - def test_namedtuple_dumps(self): - for v in [Value(1), Point(1, 2), DuckValue(1), DuckPoint(1, 2)]: - d = v._asdict() - self.assertEqual(d, json.loads(json.dumps(v))) - self.assertEqual( - d, - json.loads(json.dumps(v, namedtuple_as_object=True))) - self.assertEqual(d, json.loads(json.dumps(v, tuple_as_array=False))) - self.assertEqual( - d, - json.loads(json.dumps(v, namedtuple_as_object=True, - tuple_as_array=False))) - - def test_namedtuple_dumps_false(self): - for v in [Value(1), Point(1, 2)]: - l = list(v) - self.assertEqual( - l, - json.loads(json.dumps(v, namedtuple_as_object=False))) - self.assertRaises(TypeError, json.dumps, v, - tuple_as_array=False, namedtuple_as_object=False) - - def test_namedtuple_dump(self): - for v in [Value(1), Point(1, 2), DuckValue(1), DuckPoint(1, 2)]: - d = v._asdict() - sio = StringIO() - json.dump(v, sio) - self.assertEqual(d, json.loads(sio.getvalue())) - sio = StringIO() - json.dump(v, sio, namedtuple_as_object=True) - self.assertEqual( - d, - json.loads(sio.getvalue())) - sio = StringIO() - json.dump(v, sio, tuple_as_array=False) - self.assertEqual(d, json.loads(sio.getvalue())) - sio = StringIO() - json.dump(v, sio, namedtuple_as_object=True, - tuple_as_array=False) - self.assertEqual( - d, - json.loads(sio.getvalue())) - - def test_namedtuple_dump_false(self): - for v in [Value(1), Point(1, 2)]: - l = list(v) - sio = StringIO() - json.dump(v, sio, namedtuple_as_object=False) - self.assertEqual( - l, - json.loads(sio.getvalue())) - self.assertRaises(TypeError, json.dump, v, StringIO(), - tuple_as_array=False, namedtuple_as_object=False) - - def test_asdict_not_callable_dump(self): - for f in CONSTRUCTORS: - self.assertRaises( - TypeError, - json.dump, - f(DeadDuck()), - StringIO(), - namedtuple_as_object=True - ) - sio = StringIO() - json.dump(f(DeadDict()), sio, namedtuple_as_object=True) - self.assertEqual( - json.dumps(f({})), - sio.getvalue()) - self.assertRaises( - TypeError, - json.dump, - f(Value), - StringIO(), - namedtuple_as_object=True - ) - - def test_asdict_not_callable_dumps(self): - for f in CONSTRUCTORS: - self.assertRaises(TypeError, - json.dumps, f(DeadDuck()), namedtuple_as_object=True) - self.assertRaises( - TypeError, - json.dumps, - f(Value), - namedtuple_as_object=True - ) - self.assertEqual( - json.dumps(f({})), - json.dumps(f(DeadDict()), namedtuple_as_object=True)) - - def test_asdict_unbound_method_dumps(self): - for f in CONSTRUCTORS: - self.assertEqual( - json.dumps(f(Value), default=lambda v: v.__name__), - json.dumps(f(Value.__name__)) - ) - - def test_asdict_does_not_return_dict(self): - if not mock: - if hasattr(unittest, "SkipTest"): - raise unittest.SkipTest("unittest.mock required") - else: - print("unittest.mock not available") - return - fake = mock.Mock() - self.assertTrue(hasattr(fake, '_asdict')) - self.assertTrue(callable(fake._asdict)) - self.assertFalse(isinstance(fake._asdict(), dict)) - # https://github.com/simplejson/simplejson/pull/284 - # when running under a debug build of CPython (COPTS=-UNDEBUG) - # a C assertion could fire due to an unchecked error of an PyDict - # API call on a non-dict internally in _speedups.c. Without a debug - # build of CPython this test likely passes either way despite the - # potential for internal data corruption. Getting it to crash in - # a debug build is not always easy either as it requires an - # assert(!PyErr_Occurred()) that could fire later on. - with self.assertRaises(TypeError): - json.dumps({23: fake}, namedtuple_as_object=True, for_json=False) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_pass1.py b/contrib/python/simplejson/py2/simplejson/tests/test_pass1.py deleted file mode 100644 index f0b5b10e761..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_pass1.py +++ /dev/null @@ -1,71 +0,0 @@ -from unittest import TestCase - -import simplejson as json - -# from http://json.org/JSON_checker/test/pass1.json -JSON = r''' -[ - "JSON Test Pattern pass1", - {"object with 1 member":["array with 1 element"]}, - {}, - [], - -42, - true, - false, - null, - { - "integer": 1234567890, - "real": -9876.543210, - "e": 0.123456789e-12, - "E": 1.234567890E+34, - "": 23456789012E66, - "zero": 0, - "one": 1, - "space": " ", - "quote": "\"", - "backslash": "\\", - "controls": "\b\f\n\r\t", - "slash": "/ & \/", - "alpha": "abcdefghijklmnopqrstuvwyz", - "ALPHA": "ABCDEFGHIJKLMNOPQRSTUVWYZ", - "digit": "0123456789", - "special": "`1~!@#$%^&*()_+-={':[,]}|;.</>?", - "hex": "\u0123\u4567\u89AB\uCDEF\uabcd\uef4A", - "true": true, - "false": false, - "null": null, - "array":[ ], - "object":{ }, - "address": "50 St. James Street", - "url": "http://www.JSON.org/", - "comment": "// /* <!-- --", - "# -- --> */": " ", - " s p a c e d " :[1,2 , 3 - -, - -4 , 5 , 6 ,7 ],"compact": [1,2,3,4,5,6,7], - "jsontext": "{\"object with 1 member\":[\"array with 1 element\"]}", - "quotes": "" \u0022 %22 0x22 034 "", - "\/\\\"\uCAFE\uBABE\uAB98\uFCDE\ubcda\uef4A\b\f\n\r\t`1~!@#$%^&*()_+-=[]{}|;:',./<>?" -: "A key can be any string" - }, - 0.5 ,98.6 -, -99.44 -, - -1066, -1e1, -0.1e1, -1e-1, -1e00,2e+00,2e-00 -,"rosebud"] -''' - -class TestPass1(TestCase): - def test_parse(self): - # test in/out equivalence and parsing - res = json.loads(JSON) - out = json.dumps(res) - self.assertEqual(res, json.loads(out)) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_pass2.py b/contrib/python/simplejson/py2/simplejson/tests/test_pass2.py deleted file mode 100644 index 5d812b3bbe2..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_pass2.py +++ /dev/null @@ -1,14 +0,0 @@ -from unittest import TestCase -import simplejson as json - -# from http://json.org/JSON_checker/test/pass2.json -JSON = r''' -[[[[[[[[[[[[[[[[[[["Not too deep"]]]]]]]]]]]]]]]]]]] -''' - -class TestPass2(TestCase): - def test_parse(self): - # test in/out equivalence and parsing - res = json.loads(JSON) - out = json.dumps(res) - self.assertEqual(res, json.loads(out)) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_pass3.py b/contrib/python/simplejson/py2/simplejson/tests/test_pass3.py deleted file mode 100644 index 821d60b22cc..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_pass3.py +++ /dev/null @@ -1,20 +0,0 @@ -from unittest import TestCase - -import simplejson as json - -# from http://json.org/JSON_checker/test/pass3.json -JSON = r''' -{ - "JSON Test Pattern pass3": { - "The outermost value": "must be an object or array.", - "In this test": "It is an object." - } -} -''' - -class TestPass3(TestCase): - def test_parse(self): - # test in/out equivalence and parsing - res = json.loads(JSON) - out = json.dumps(res) - self.assertEqual(res, json.loads(out)) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_raw_json.py b/contrib/python/simplejson/py2/simplejson/tests/test_raw_json.py deleted file mode 100644 index 1dfcc2c5f95..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_raw_json.py +++ /dev/null @@ -1,47 +0,0 @@ -import unittest -import simplejson as json - -dct1 = { - 'key1': 'value1' -} - -dct2 = { - 'key2': 'value2', - 'd1': dct1 -} - -dct3 = { - 'key2': 'value2', - 'd1': json.dumps(dct1) -} - -dct4 = { - 'key2': 'value2', - 'd1': json.RawJSON(json.dumps(dct1)) -} - - -class TestRawJson(unittest.TestCase): - - def test_normal_str(self): - self.assertNotEqual(json.dumps(dct2), json.dumps(dct3)) - - def test_raw_json_str(self): - self.assertEqual(json.dumps(dct2), json.dumps(dct4)) - self.assertEqual(dct2, json.loads(json.dumps(dct4))) - - def test_list(self): - self.assertEqual( - json.dumps([dct2]), - json.dumps([json.RawJSON(json.dumps(dct2))])) - self.assertEqual( - [dct2], - json.loads(json.dumps([json.RawJSON(json.dumps(dct2))]))) - - def test_direct(self): - self.assertEqual( - json.dumps(dct2), - json.dumps(json.RawJSON(json.dumps(dct2)))) - self.assertEqual( - dct2, - json.loads(json.dumps(json.RawJSON(json.dumps(dct2))))) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_recursion.py b/contrib/python/simplejson/py2/simplejson/tests/test_recursion.py deleted file mode 100644 index 662eb667ece..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_recursion.py +++ /dev/null @@ -1,67 +0,0 @@ -from unittest import TestCase - -import simplejson as json - -class JSONTestObject: - pass - - -class RecursiveJSONEncoder(json.JSONEncoder): - recurse = False - def default(self, o): - if o is JSONTestObject: - if self.recurse: - return [JSONTestObject] - else: - return 'JSONTestObject' - return json.JSONEncoder.default(o) - - -class TestRecursion(TestCase): - def test_listrecursion(self): - x = [] - x.append(x) - try: - json.dumps(x) - except ValueError: - pass - else: - self.fail("didn't raise ValueError on list recursion") - x = [] - y = [x] - x.append(y) - try: - json.dumps(x) - except ValueError: - pass - else: - self.fail("didn't raise ValueError on alternating list recursion") - y = [] - x = [y, y] - # ensure that the marker is cleared - json.dumps(x) - - def test_dictrecursion(self): - x = {} - x["test"] = x - try: - json.dumps(x) - except ValueError: - pass - else: - self.fail("didn't raise ValueError on dict recursion") - x = {} - y = {"a": x, "b": x} - # ensure that the marker is cleared - json.dumps(y) - - def test_defaultrecursion(self): - enc = RecursiveJSONEncoder() - self.assertEqual(enc.encode(JSONTestObject), '"JSONTestObject"') - enc.recurse = True - try: - enc.encode(JSONTestObject) - except ValueError: - pass - else: - self.fail("didn't raise ValueError on default recursion") diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_scanstring.py b/contrib/python/simplejson/py2/simplejson/tests/test_scanstring.py deleted file mode 100644 index 1f544834209..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_scanstring.py +++ /dev/null @@ -1,200 +0,0 @@ -import sys -from unittest import TestCase - -import simplejson as json -import simplejson.decoder -from simplejson.compat import b, PY3 - -class TestScanString(TestCase): - # The bytes type is intentionally not used in most of these tests - # under Python 3 because the decoder immediately coerces to str before - # calling scanstring. In Python 2 we are testing the code paths - # for both unicode and str. - # - # The reason this is done is because Python 3 would require - # entirely different code paths for parsing bytes and str. - # - def test_py_scanstring(self): - self._test_scanstring(simplejson.decoder.py_scanstring) - - def test_c_scanstring(self): - if not simplejson.decoder.c_scanstring: - return - self._test_scanstring(simplejson.decoder.c_scanstring) - - self.assertTrue(isinstance(simplejson.decoder.c_scanstring('""', 0)[0], str)) - - def _test_scanstring(self, scanstring): - if sys.maxunicode == 65535: - self.assertEqual( - scanstring(u'"z\U0001d120x"', 1, None, True), - (u'z\U0001d120x', 6)) - else: - self.assertEqual( - scanstring(u'"z\U0001d120x"', 1, None, True), - (u'z\U0001d120x', 5)) - - self.assertEqual( - scanstring('"\\u007b"', 1, None, True), - (u'{', 8)) - - self.assertEqual( - scanstring('"A JSON payload should be an object or array, not a string."', 1, None, True), - (u'A JSON payload should be an object or array, not a string.', 60)) - - self.assertEqual( - scanstring('["Unclosed array"', 2, None, True), - (u'Unclosed array', 17)) - - self.assertEqual( - scanstring('["extra comma",]', 2, None, True), - (u'extra comma', 14)) - - self.assertEqual( - scanstring('["double extra comma",,]', 2, None, True), - (u'double extra comma', 21)) - - self.assertEqual( - scanstring('["Comma after the close"],', 2, None, True), - (u'Comma after the close', 24)) - - self.assertEqual( - scanstring('["Extra close"]]', 2, None, True), - (u'Extra close', 14)) - - self.assertEqual( - scanstring('{"Extra comma": true,}', 2, None, True), - (u'Extra comma', 14)) - - self.assertEqual( - scanstring('{"Extra value after close": true} "misplaced quoted value"', 2, None, True), - (u'Extra value after close', 26)) - - self.assertEqual( - scanstring('{"Illegal expression": 1 + 2}', 2, None, True), - (u'Illegal expression', 21)) - - self.assertEqual( - scanstring('{"Illegal invocation": alert()}', 2, None, True), - (u'Illegal invocation', 21)) - - self.assertEqual( - scanstring('{"Numbers cannot have leading zeroes": 013}', 2, None, True), - (u'Numbers cannot have leading zeroes', 37)) - - self.assertEqual( - scanstring('{"Numbers cannot be hex": 0x14}', 2, None, True), - (u'Numbers cannot be hex', 24)) - - self.assertEqual( - scanstring('[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]', 21, None, True), - (u'Too deep', 30)) - - self.assertEqual( - scanstring('{"Missing colon" null}', 2, None, True), - (u'Missing colon', 16)) - - self.assertEqual( - scanstring('{"Double colon":: null}', 2, None, True), - (u'Double colon', 15)) - - self.assertEqual( - scanstring('{"Comma instead of colon", null}', 2, None, True), - (u'Comma instead of colon', 25)) - - self.assertEqual( - scanstring('["Colon instead of comma": false]', 2, None, True), - (u'Colon instead of comma', 25)) - - self.assertEqual( - scanstring('["Bad value", truth]', 2, None, True), - (u'Bad value', 12)) - - for c in map(chr, range(0x00, 0x1f)): - self.assertEqual( - scanstring(c + '"', 0, None, False), - (c, 2)) - self.assertRaises( - ValueError, - scanstring, c + '"', 0, None, True) - - self.assertRaises(ValueError, scanstring, '', 0, None, True) - self.assertRaises(ValueError, scanstring, 'a', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\u', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\u0', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\u01', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\u012', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\u0123', 0, None, True) - if sys.maxunicode > 65535: - self.assertRaises(ValueError, - scanstring, '\\ud834\\u"', 0, None, True) - self.assertRaises(ValueError, - scanstring, '\\ud834\\x0123"', 0, None, True) - - self.assertRaises(json.JSONDecodeError, scanstring, '\\u-123"', 0, None, True) - # SJ-PT-23-01: Invalid Handling of Broken Unicode Escape Sequences - self.assertRaises(json.JSONDecodeError, scanstring, '\\u EDD"', 0, None, True) - - def test_issue3623(self): - self.assertRaises(ValueError, json.decoder.scanstring, "xxx", 1, - "xxx") - self.assertRaises(UnicodeDecodeError, - json.encoder.encode_basestring_ascii, b("xx\xff")) - - def test_overflow(self): - # Python 2.5 does not have maxsize, Python 3 does not have maxint - maxsize = getattr(sys, 'maxsize', getattr(sys, 'maxint', None)) - assert maxsize is not None - self.assertRaises(OverflowError, json.decoder.scanstring, "xxx", - maxsize + 1) - - def test_surrogates(self): - scanstring = json.decoder.scanstring - - def assertScan(given, expect, test_utf8=True): - givens = [given] - if not PY3 and test_utf8: - givens.append(given.encode('utf8')) - for given in givens: - (res, count) = scanstring(given, 1, None, True) - self.assertEqual(len(given), count) - self.assertEqual(res, expect) - - assertScan( - u'"z\\ud834\\u0079x"', - u'z\ud834yx') - assertScan( - u'"z\\ud834\\udd20x"', - u'z\U0001d120x') - assertScan( - u'"z\\ud834\\ud834\\udd20x"', - u'z\ud834\U0001d120x') - assertScan( - u'"z\\ud834x"', - u'z\ud834x') - assertScan( - u'"z\\udd20x"', - u'z\udd20x') - assertScan( - u'"z\ud834x"', - u'z\ud834x') - # It may look strange to join strings together, but Python is drunk. - # https://gist.github.com/etrepum/5538443 - assertScan( - u'"z\\ud834\udd20x12345"', - u''.join([u'z\ud834', u'\udd20x12345'])) - assertScan( - u'"z\ud834\\udd20x"', - u''.join([u'z\ud834', u'\udd20x'])) - # these have different behavior given UTF8 input, because the surrogate - # pair may be joined (in maxunicode > 65535 builds) - assertScan( - u''.join([u'"z\ud834', u'\udd20x"']), - u''.join([u'z\ud834', u'\udd20x']), - test_utf8=False) - - self.assertRaises(ValueError, - scanstring, u'"z\\ud83x"', 1, None, True) - self.assertRaises(ValueError, - scanstring, u'"z\\ud834\\udd2x"', 1, None, True) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_separators.py b/contrib/python/simplejson/py2/simplejson/tests/test_separators.py deleted file mode 100644 index 91b4d4fb6f3..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_separators.py +++ /dev/null @@ -1,42 +0,0 @@ -import textwrap -from unittest import TestCase - -import simplejson as json - - -class TestSeparators(TestCase): - def test_separators(self): - h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', 'i-vhbjkhnth', - {'nifty': 87}, {'field': 'yes', 'morefield': False} ] - - expect = textwrap.dedent("""\ - [ - [ - "blorpie" - ] , - [ - "whoops" - ] , - [] , - "d-shtaeou" , - "d-nthiouh" , - "i-vhbjkhnth" , - { - "nifty" : 87 - } , - { - "field" : "yes" , - "morefield" : false - } - ]""") - - - d1 = json.dumps(h) - d2 = json.dumps(h, indent=' ', sort_keys=True, separators=(' ,', ' : ')) - - h1 = json.loads(d1) - h2 = json.loads(d2) - - self.assertEqual(h1, h) - self.assertEqual(h2, h) - self.assertEqual(d2, expect) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_speedups.py b/contrib/python/simplejson/py2/simplejson/tests/test_speedups.py deleted file mode 100644 index 8b146df9a05..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_speedups.py +++ /dev/null @@ -1,114 +0,0 @@ -from __future__ import with_statement - -import sys -import unittest -from unittest import TestCase - -import simplejson -from simplejson import encoder, decoder, scanner -from simplejson.compat import PY3, long_type, b - - -def has_speedups(): - return encoder.c_make_encoder is not None - - -def skip_if_speedups_missing(func): - def wrapper(*args, **kwargs): - if not has_speedups(): - if hasattr(unittest, 'SkipTest'): - raise unittest.SkipTest("C Extension not available") - else: - sys.stdout.write("C Extension not available") - return - return func(*args, **kwargs) - - return wrapper - - -class BadBool: - def __bool__(self): - 1/0 - __nonzero__ = __bool__ - - -class TestDecode(TestCase): - @skip_if_speedups_missing - def test_make_scanner(self): - self.assertRaises(AttributeError, scanner.c_make_scanner, 1) - - @skip_if_speedups_missing - def test_bad_bool_args(self): - def test(value): - decoder.JSONDecoder(strict=BadBool()).decode(value) - self.assertRaises(ZeroDivisionError, test, '""') - self.assertRaises(ZeroDivisionError, test, '{}') - if not PY3: - self.assertRaises(ZeroDivisionError, test, u'""') - self.assertRaises(ZeroDivisionError, test, u'{}') - -class TestEncode(TestCase): - @skip_if_speedups_missing - def test_make_encoder(self): - self.assertRaises( - TypeError, - encoder.c_make_encoder, - None, - ("\xCD\x7D\x3D\x4E\x12\x4C\xF9\x79\xD7" - "\x52\xBA\x82\xF2\x27\x4A\x7D\xA0\xCA\x75"), - None - ) - - @skip_if_speedups_missing - def test_bad_str_encoder(self): - # Issue #31505: There shouldn't be an assertion failure in case - # c_make_encoder() receives a bad encoder() argument. - import decimal - def bad_encoder1(*args): - return None - enc = encoder.c_make_encoder( - None, lambda obj: str(obj), - bad_encoder1, None, ': ', ', ', - False, False, False, {}, False, False, False, - None, None, 'utf-8', False, False, decimal.Decimal, False) - self.assertRaises(TypeError, enc, 'spam', 4) - self.assertRaises(TypeError, enc, {'spam': 42}, 4) - - def bad_encoder2(*args): - 1/0 - enc = encoder.c_make_encoder( - None, lambda obj: str(obj), - bad_encoder2, None, ': ', ', ', - False, False, False, {}, False, False, False, - None, None, 'utf-8', False, False, decimal.Decimal, False) - self.assertRaises(ZeroDivisionError, enc, 'spam', 4) - - @skip_if_speedups_missing - def test_bad_bool_args(self): - def test(name): - encoder.JSONEncoder(**{name: BadBool()}).encode({}) - self.assertRaises(ZeroDivisionError, test, 'skipkeys') - self.assertRaises(ZeroDivisionError, test, 'ensure_ascii') - self.assertRaises(ZeroDivisionError, test, 'check_circular') - self.assertRaises(ZeroDivisionError, test, 'allow_nan') - self.assertRaises(ZeroDivisionError, test, 'sort_keys') - self.assertRaises(ZeroDivisionError, test, 'use_decimal') - self.assertRaises(ZeroDivisionError, test, 'namedtuple_as_object') - self.assertRaises(ZeroDivisionError, test, 'tuple_as_array') - self.assertRaises(ZeroDivisionError, test, 'bigint_as_string') - self.assertRaises(ZeroDivisionError, test, 'for_json') - self.assertRaises(ZeroDivisionError, test, 'ignore_nan') - self.assertRaises(ZeroDivisionError, test, 'iterable_as_array') - - @skip_if_speedups_missing - def test_int_as_string_bitcount_overflow(self): - long_count = long_type(2)**32+31 - def test(): - encoder.JSONEncoder(int_as_string_bitcount=long_count).encode(0) - self.assertRaises((TypeError, OverflowError), test) - - if PY3: - @skip_if_speedups_missing - def test_bad_encoding(self): - with self.assertRaises(UnicodeEncodeError): - encoder.JSONEncoder(encoding='\udcff').encode({b('key'): 123}) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_str_subclass.py b/contrib/python/simplejson/py2/simplejson/tests/test_str_subclass.py deleted file mode 100644 index b6c8351f264..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_str_subclass.py +++ /dev/null @@ -1,21 +0,0 @@ -from unittest import TestCase - -import simplejson -from simplejson.compat import text_type - -# Tests for issue demonstrated in https://github.com/simplejson/simplejson/issues/144 -class WonkyTextSubclass(text_type): - def __getslice__(self, start, end): - return self.__class__('not what you wanted!') - -class TestStrSubclass(TestCase): - def test_dump_load(self): - for s in ['', '"hello"', 'text', u'\u005c']: - self.assertEqual( - s, - simplejson.loads(simplejson.dumps(WonkyTextSubclass(s)))) - - self.assertEqual( - s, - simplejson.loads(simplejson.dumps(WonkyTextSubclass(s), - ensure_ascii=False))) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_subclass.py b/contrib/python/simplejson/py2/simplejson/tests/test_subclass.py deleted file mode 100644 index 2bae3b6309e..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_subclass.py +++ /dev/null @@ -1,37 +0,0 @@ -from unittest import TestCase -import simplejson as json - -from decimal import Decimal - -class AlternateInt(int): - def __repr__(self): - return 'invalid json' - __str__ = __repr__ - - -class AlternateFloat(float): - def __repr__(self): - return 'invalid json' - __str__ = __repr__ - - -# class AlternateDecimal(Decimal): -# def __repr__(self): -# return 'invalid json' - - -class TestSubclass(TestCase): - def test_int(self): - self.assertEqual(json.dumps(AlternateInt(1)), '1') - self.assertEqual(json.dumps(AlternateInt(-1)), '-1') - self.assertEqual(json.loads(json.dumps({AlternateInt(1): 1})), {'1': 1}) - - def test_float(self): - self.assertEqual(json.dumps(AlternateFloat(1.0)), '1.0') - self.assertEqual(json.dumps(AlternateFloat(-1.0)), '-1.0') - self.assertEqual(json.loads(json.dumps({AlternateFloat(1.0): 1})), {'1.0': 1}) - - # NOTE: Decimal subclasses are not supported as-is - # def test_decimal(self): - # self.assertEqual(json.dumps(AlternateDecimal('1.0')), '1.0') - # self.assertEqual(json.dumps(AlternateDecimal('-1.0')), '-1.0') diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_tool.py b/contrib/python/simplejson/py2/simplejson/tests/test_tool.py deleted file mode 100644 index 914bff8bca5..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_tool.py +++ /dev/null @@ -1,114 +0,0 @@ -from __future__ import with_statement -import os -import sys -import textwrap -import unittest -import subprocess -import tempfile -try: - # Python 3.x - from test.support import strip_python_stderr -except ImportError: - # Python 2.6+ - try: - from test.test_support import strip_python_stderr - except ImportError: - # Python 2.5 - import re - def strip_python_stderr(stderr): - return re.sub( - r"\[\d+ refs\]\r?\n?$".encode(), - "".encode(), - stderr).strip() - -def open_temp_file(): - if sys.version_info >= (2, 6): - file = tempfile.NamedTemporaryFile(delete=False) - filename = file.name - else: - fd, filename = tempfile.mkstemp() - file = os.fdopen(fd, 'w+b') - return file, filename - -class TestTool(unittest.TestCase): - data = """ - - [["blorpie"],[ "whoops" ] , [ - ],\t"d-shtaeou",\r"d-nthiouh", - "i-vhbjkhnth", {"nifty":87}, {"morefield" :\tfalse,"field" - :"yes"} ] - """ - - expect = textwrap.dedent("""\ - [ - [ - "blorpie" - ], - [ - "whoops" - ], - [], - "d-shtaeou", - "d-nthiouh", - "i-vhbjkhnth", - { - "nifty": 87 - }, - { - "field": "yes", - "morefield": false - } - ] - """) - - def runTool(self, args=None, data=None): - argv = [sys.executable, '-m', 'simplejson.tool'] - if args: - argv.extend(args) - proc = subprocess.Popen(argv, - stdin=subprocess.PIPE, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE) - out, err = proc.communicate(data) - self.assertEqual(strip_python_stderr(err), ''.encode()) - self.assertEqual(proc.returncode, 0) - return out.decode('utf8').splitlines() - - def test_stdin_stdout(self): - self.assertEqual( - self.runTool(data=self.data.encode()), - self.expect.splitlines()) - - def test_infile_stdout(self): - infile, infile_name = open_temp_file() - try: - infile.write(self.data.encode()) - infile.close() - self.assertEqual( - self.runTool(args=[infile_name]), - self.expect.splitlines()) - finally: - os.unlink(infile_name) - - def test_infile_outfile(self): - infile, infile_name = open_temp_file() - try: - infile.write(self.data.encode()) - infile.close() - # outfile will get overwritten by tool, so the delete - # may not work on some platforms. Do it manually. - outfile, outfile_name = open_temp_file() - try: - outfile.close() - self.assertEqual( - self.runTool(args=[infile_name, outfile_name]), - []) - with open(outfile_name, 'rb') as f: - self.assertEqual( - f.read().decode('utf8').splitlines(), - self.expect.splitlines() - ) - finally: - os.unlink(outfile_name) - finally: - os.unlink(infile_name) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_tuple.py b/contrib/python/simplejson/py2/simplejson/tests/test_tuple.py deleted file mode 100644 index 4ad7b0e861b..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_tuple.py +++ /dev/null @@ -1,47 +0,0 @@ -import unittest - -from simplejson.compat import StringIO -import simplejson as json - -class TestTuples(unittest.TestCase): - def test_tuple_array_dumps(self): - t = (1, 2, 3) - expect = json.dumps(list(t)) - # Default is True - self.assertEqual(expect, json.dumps(t)) - self.assertEqual(expect, json.dumps(t, tuple_as_array=True)) - self.assertRaises(TypeError, json.dumps, t, tuple_as_array=False) - # Ensure that the "default" does not get called - self.assertEqual(expect, json.dumps(t, default=repr)) - self.assertEqual(expect, json.dumps(t, tuple_as_array=True, - default=repr)) - # Ensure that the "default" gets called - self.assertEqual( - json.dumps(repr(t)), - json.dumps(t, tuple_as_array=False, default=repr)) - - def test_tuple_array_dump(self): - t = (1, 2, 3) - expect = json.dumps(list(t)) - # Default is True - sio = StringIO() - json.dump(t, sio) - self.assertEqual(expect, sio.getvalue()) - sio = StringIO() - json.dump(t, sio, tuple_as_array=True) - self.assertEqual(expect, sio.getvalue()) - self.assertRaises(TypeError, json.dump, t, StringIO(), - tuple_as_array=False) - # Ensure that the "default" does not get called - sio = StringIO() - json.dump(t, sio, default=repr) - self.assertEqual(expect, sio.getvalue()) - sio = StringIO() - json.dump(t, sio, tuple_as_array=True, default=repr) - self.assertEqual(expect, sio.getvalue()) - # Ensure that the "default" gets called - sio = StringIO() - json.dump(t, sio, tuple_as_array=False, default=repr) - self.assertEqual( - json.dumps(repr(t)), - sio.getvalue()) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_unicode.py b/contrib/python/simplejson/py2/simplejson/tests/test_unicode.py deleted file mode 100644 index 0c7b1a6c444..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_unicode.py +++ /dev/null @@ -1,154 +0,0 @@ -import sys -import codecs -from unittest import TestCase - -import simplejson as json -from simplejson.compat import unichr, text_type, b, BytesIO - -class TestUnicode(TestCase): - def test_encoding1(self): - encoder = json.JSONEncoder(encoding='utf-8') - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - s = u.encode('utf-8') - ju = encoder.encode(u) - js = encoder.encode(s) - self.assertEqual(ju, js) - - def test_encoding2(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - s = u.encode('utf-8') - ju = json.dumps(u, encoding='utf-8') - js = json.dumps(s, encoding='utf-8') - self.assertEqual(ju, js) - - def test_encoding3(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps(u) - self.assertEqual(j, '"\\u03b1\\u03a9"') - - def test_encoding4(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps([u]) - self.assertEqual(j, '["\\u03b1\\u03a9"]') - - def test_encoding5(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps(u, ensure_ascii=False) - self.assertEqual(j, u'"' + u + u'"') - - def test_encoding6(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps([u], ensure_ascii=False) - self.assertEqual(j, u'["' + u + u'"]') - - def test_big_unicode_encode(self): - u = u'\U0001d120' - self.assertEqual(json.dumps(u), '"\\ud834\\udd20"') - self.assertEqual(json.dumps(u, ensure_ascii=False), u'"\U0001d120"') - - def test_big_unicode_decode(self): - u = u'z\U0001d120x' - self.assertEqual(json.loads('"' + u + '"'), u) - self.assertEqual(json.loads('"z\\ud834\\udd20x"'), u) - - def test_unicode_decode(self): - for i in range(0, 0xd7ff): - u = unichr(i) - #s = '"\\u{0:04x}"'.format(i) - s = '"\\u%04x"' % (i,) - self.assertEqual(json.loads(s), u) - - def test_object_pairs_hook_with_unicode(self): - s = u'{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}' - p = [(u"xkd", 1), (u"kcw", 2), (u"art", 3), (u"hxm", 4), - (u"qrt", 5), (u"pad", 6), (u"hoy", 7)] - self.assertEqual(json.loads(s), eval(s)) - self.assertEqual(json.loads(s, object_pairs_hook=lambda x: x), p) - od = json.loads(s, object_pairs_hook=json.OrderedDict) - self.assertEqual(od, json.OrderedDict(p)) - self.assertEqual(type(od), json.OrderedDict) - # the object_pairs_hook takes priority over the object_hook - self.assertEqual(json.loads(s, - object_pairs_hook=json.OrderedDict, - object_hook=lambda x: None), - json.OrderedDict(p)) - - - def test_default_encoding(self): - self.assertEqual(json.loads(u'{"a": "\xe9"}'.encode('utf-8')), - {'a': u'\xe9'}) - - def test_unicode_preservation(self): - self.assertEqual(type(json.loads(u'""')), text_type) - self.assertEqual(type(json.loads(u'"a"')), text_type) - self.assertEqual(type(json.loads(u'["a"]')[0]), text_type) - - def test_ensure_ascii_false_returns_unicode(self): - # http://code.google.com/p/simplejson/issues/detail?id=48 - self.assertEqual(type(json.dumps([], ensure_ascii=False)), text_type) - self.assertEqual(type(json.dumps(0, ensure_ascii=False)), text_type) - self.assertEqual(type(json.dumps({}, ensure_ascii=False)), text_type) - self.assertEqual(type(json.dumps("", ensure_ascii=False)), text_type) - - def test_ensure_ascii_false_bytestring_encoding(self): - # http://code.google.com/p/simplejson/issues/detail?id=48 - doc1 = {u'quux': b('Arr\xc3\xaat sur images')} - doc2 = {u'quux': u'Arr\xeat sur images'} - doc_ascii = '{"quux": "Arr\\u00eat sur images"}' - doc_unicode = u'{"quux": "Arr\xeat sur images"}' - self.assertEqual(json.dumps(doc1), doc_ascii) - self.assertEqual(json.dumps(doc2), doc_ascii) - self.assertEqual(json.dumps(doc1, ensure_ascii=False), doc_unicode) - self.assertEqual(json.dumps(doc2, ensure_ascii=False), doc_unicode) - - def test_ensure_ascii_linebreak_encoding(self): - # http://timelessrepo.com/json-isnt-a-javascript-subset - s1 = u'\u2029\u2028' - s2 = s1.encode('utf8') - expect = '"\\u2029\\u2028"' - expect_non_ascii = u'"\u2029\u2028"' - self.assertEqual(json.dumps(s1), expect) - self.assertEqual(json.dumps(s2), expect) - self.assertEqual(json.dumps(s1, ensure_ascii=False), expect_non_ascii) - self.assertEqual(json.dumps(s2, ensure_ascii=False), expect_non_ascii) - - def test_invalid_escape_sequences(self): - # incomplete escape sequence - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u1') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u12') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u123') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u1234') - # invalid escape sequence - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u123x"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u12x4"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u1x34"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ux234"') - if sys.maxunicode > 65535: - # invalid escape sequence for low surrogate - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u0"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u00"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u000"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u000x"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u00x0"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u0x00"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\ux000"') - - def test_ensure_ascii_still_works(self): - # in the ascii range, ensure that everything is the same - for c in map(unichr, range(0, 127)): - self.assertEqual( - json.dumps(c, ensure_ascii=False), - json.dumps(c)) - snowman = u'\N{SNOWMAN}' - self.assertEqual( - json.dumps(c, ensure_ascii=False), - '"' + c + '"') - - def test_strip_bom(self): - content = u"\u3053\u3093\u306b\u3061\u308f" - json_doc = codecs.BOM_UTF8 + b(json.dumps(content)) - self.assertEqual(json.load(BytesIO(json_doc)), content) - for doc in json_doc, json_doc.decode('utf8'): - self.assertEqual(json.loads(doc), content) diff --git a/contrib/python/simplejson/py2/simplejson/tool.py b/contrib/python/simplejson/py2/simplejson/tool.py deleted file mode 100644 index 062e8e2c181..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tool.py +++ /dev/null @@ -1,42 +0,0 @@ -r"""Command-line tool to validate and pretty-print JSON - -Usage:: - - $ echo '{"json":"obj"}' | python -m simplejson.tool - { - "json": "obj" - } - $ echo '{ 1.2:3.4}' | python -m simplejson.tool - Expecting property name: line 1 column 2 (char 2) - -""" -from __future__ import with_statement -import sys -import simplejson as json - -def main(): - if len(sys.argv) == 1: - infile = sys.stdin - outfile = sys.stdout - elif len(sys.argv) == 2: - infile = open(sys.argv[1], 'r') - outfile = sys.stdout - elif len(sys.argv) == 3: - infile = open(sys.argv[1], 'r') - outfile = open(sys.argv[2], 'w') - else: - raise SystemExit(sys.argv[0] + " [infile [outfile]]") - with infile: - try: - obj = json.load(infile, - object_pairs_hook=json.OrderedDict, - use_decimal=True) - except ValueError: - raise SystemExit(sys.exc_info()[1]) - with outfile: - json.dump(obj, outfile, sort_keys=True, indent=' ', use_decimal=True) - outfile.write('\n') - - -if __name__ == '__main__': - main() diff --git a/contrib/python/simplejson/py2/tests/ya.make b/contrib/python/simplejson/py2/tests/ya.make deleted file mode 100644 index 23e25312213..00000000000 --- a/contrib/python/simplejson/py2/tests/ya.make +++ /dev/null @@ -1,47 +0,0 @@ -PY2TEST() - -PEERDIR( - contrib/python/simplejson -) - -SRCDIR( - contrib/python/simplejson/py2/simplejson/tests -) - -TEST_SRCS( - __init__.py - test_bigint_as_string.py - test_bitsize_int_as_string.py - test_check_circular.py - test_decimal.py - test_decode.py - test_default.py - test_dump.py - test_encode_basestring_ascii.py - test_encode_for_html.py - test_errors.py - test_fail.py - test_float.py - test_for_json.py - test_indent.py - test_item_sort_key.py - test_iterable.py - test_namedtuple.py - test_pass1.py - test_pass2.py - test_pass3.py - test_raw_json.py - test_recursion.py - test_scanstring.py - test_separators.py - test_speedups.py - test_str_subclass.py - test_subclass.py - # test_tool.py - test_tuple.py - test_unicode.py -) - -NO_LINT() - -END() diff --git a/contrib/python/simplejson/py2/ya.make b/contrib/python/simplejson/py2/ya.make deleted file mode 100644 index 27c6bf9e27f..00000000000 --- a/contrib/python/simplejson/py2/ya.make +++ /dev/null @@ -1,48 +0,0 @@ -# Generated by devtools/yamaker (pypi). - -PY2_LIBRARY() - -VERSION(3.20.2) - -LICENSE(MIT) - -NO_COMPILER_WARNINGS() - -NO_LINT() - -NO_CHECK_IMPORTS( - simplejson.ordered_dict -) - -SRCS( - simplejson/_speedups.c -) - -PY_REGISTER( - simplejson._speedups -) - -PY_SRCS( - TOP_LEVEL - simplejson/__init__.py - simplejson/compat.py - simplejson/decoder.py - simplejson/encoder.py - simplejson/errors.py - simplejson/ordered_dict.py - simplejson/raw_json.py - simplejson/scanner.py - simplejson/tool.py -) - -RESOURCE_FILES( - PREFIX contrib/python/simplejson/py2/ - .dist-info/METADATA - .dist-info/top_level.txt -) - -END() - -RECURSE_FOR_TESTS( - tests -) diff --git a/contrib/python/simplejson/py3/.dist-info/METADATA b/contrib/python/simplejson/py3/.dist-info/METADATA deleted file mode 100644 index 31a275ebf8c..00000000000 --- a/contrib/python/simplejson/py3/.dist-info/METADATA +++ /dev/null @@ -1,80 +0,0 @@ -Metadata-Version: 2.4 -Name: simplejson -Version: 3.20.2 -Summary: Simple, fast, extensible JSON encoder/decoder for Python -Home-page: https://github.com/simplejson/simplejson -Author: Bob Ippolito -Author-email: [email protected] -License: MIT License -Platform: any -Classifier: Development Status :: 5 - Production/Stable -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: MIT License -Classifier: License :: OSI Approved :: Academic Free License (AFL) -Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 2 -Classifier: Programming Language :: Python :: 2.5 -Classifier: Programming Language :: Python :: 2.6 -Classifier: Programming Language :: Python :: 2.7 -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.3 -Classifier: Programming Language :: Python :: 3.4 -Classifier: Programming Language :: Python :: 3.5 -Classifier: Programming Language :: Python :: 3.6 -Classifier: Programming Language :: Python :: 3.7 -Classifier: Programming Language :: Python :: 3.8 -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Programming Language :: Python :: Implementation :: CPython -Classifier: Programming Language :: Python :: Implementation :: PyPy -Classifier: Topic :: Software Development :: Libraries :: Python Modules -Requires-Python: >=2.5, !=3.0.*, !=3.1.*, !=3.2.* -License-File: LICENSE.txt -Dynamic: author -Dynamic: author-email -Dynamic: classifier -Dynamic: description -Dynamic: home-page -Dynamic: license -Dynamic: license-file -Dynamic: platform -Dynamic: requires-python -Dynamic: summary - -simplejson ----------- - -simplejson is a simple, fast, complete, correct and extensible -JSON <http://json.org> encoder and decoder for Python 3.3+ -with legacy support for Python 2.5+. It is pure Python code -with no dependencies, but includes an optional C extension -for a serious speed boost. - -The latest documentation for simplejson can be read online here: -https://simplejson.readthedocs.io/ - -simplejson is the externally maintained development version of the -json library included with Python (since 2.6). This version is tested -with the latest Python 3.8 and maintains backwards compatibility -with Python 3.3+ and the legacy Python 2.5 - Python 2.7 releases. - -The encoder can be specialized to provide serialization in any kind of -situation, without any special support by the objects to be serialized -(somewhat like pickle). This is best done with the ``default`` kwarg -to dumps. - -The decoder can handle incoming JSON strings of any specified encoding -(UTF-8 by default). It can also be specialized to post-process JSON -objects with the ``object_hook`` or ``object_pairs_hook`` kwargs. This -is particularly useful for implementing protocols such as JSON-RPC -that have a richer type system than JSON itself. - -For those of you that have legacy systems to maintain, there is a -very old fork of simplejson in the `python2.2`_ branch that supports -Python 2.2. This is based on a very old version of simplejson, -is not maintained, and should only be used as a last resort. - -.. _python2.2: https://github.com/simplejson/simplejson/tree/python2.2 diff --git a/contrib/python/simplejson/py3/.dist-info/top_level.txt b/contrib/python/simplejson/py3/.dist-info/top_level.txt deleted file mode 100644 index 322630ee75f..00000000000 --- a/contrib/python/simplejson/py3/.dist-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -simplejson diff --git a/contrib/python/simplejson/py3/LICENSE.txt b/contrib/python/simplejson/py3/LICENSE.txt deleted file mode 100644 index e05f49c3fd0..00000000000 --- a/contrib/python/simplejson/py3/LICENSE.txt +++ /dev/null @@ -1,79 +0,0 @@ -simplejson is dual-licensed software. It is available under the terms -of the MIT license, or the Academic Free License version 2.1. The full -text of each license agreement is included below. This code is also -licensed to the Python Software Foundation (PSF) under a Contributor -Agreement. - -MIT License -=========== - -Copyright (c) 2006 Bob Ippolito - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -Academic Free License v. 2.1 -============================ - -Copyright (c) 2006 Bob Ippolito. All rights reserved. - -This Academic Free License (the "License") applies to any original work of authorship (the "Original Work") whose owner (the "Licensor") has placed the following notice immediately following the copyright notice for the Original Work: - -Licensed under the Academic Free License version 2.1 - -1) Grant of Copyright License. Licensor hereby grants You a world-wide, royalty-free, non-exclusive, perpetual, sublicenseable license to do the following: - -a) to reproduce the Original Work in copies; - -b) to prepare derivative works ("Derivative Works") based upon the Original Work; - -c) to distribute copies of the Original Work and Derivative Works to the public; - -d) to perform the Original Work publicly; and - -e) to display the Original Work publicly. - -2) Grant of Patent License. Licensor hereby grants You a world-wide, royalty-free, non-exclusive, perpetual, sublicenseable license, under patent claims owned or controlled by the Licensor that are embodied in the Original Work as furnished by the Licensor, to make, use, sell and offer for sale the Original Work and Derivative Works. - -3) Grant of Source Code License. The term "Source Code" means the preferred form of the Original Work for making modifications to it and all available documentation describing how to modify the Original Work. Licensor hereby agrees to provide a machine-readable copy of the Source Code of the Original Work along with each copy of the Original Work that Licensor distributes. Licensor reserves the right to satisfy this obligation by placing a machine-readable copy of the Source Code in an information repository reasonably calculated to permit inexpensive and convenient access by You for as long as Licensor continues to distribute the Original Work, and by publishing the address of that information repository in a notice immediately following the copyright notice that applies to the Original Work. - -4) Exclusions From License Grant. Neither the names of Licensor, nor the names of any contributors to the Original Work, nor any of their trademarks or service marks, may be used to endorse or promote products derived from this Original Work without express prior written permission of the Licensor. Nothing in this License shall be deemed to grant any rights to trademarks, copyrights, patents, trade secrets or any other intellectual property of Licensor except as expressly stated herein. No patent license is granted to make, use, sell or offer to sell embodiments of any patent claims other than the licensed claims defined in Section 2. No right is granted to the trademarks of Licensor even if such marks are included in the Original Work. Nothing in this License shall be interpreted to prohibit Licensor from licensing under different terms from this License any Original Work that Licensor otherwise would have a right to license. - -5) This section intentionally omitted. - -6) Attribution Rights. You must retain, in the Source Code of any Derivative Works that You create, all copyright, patent or trademark notices from the Source Code of the Original Work, as well as any notices of licensing and any descriptive text identified therein as an "Attribution Notice." You must cause the Source Code for any Derivative Works that You create to carry a prominent Attribution Notice reasonably calculated to inform recipients that You have modified the Original Work. - -7) Warranty of Provenance and Disclaimer of Warranty. Licensor warrants that the copyright in and to the Original Work and the patent rights granted herein by Licensor are owned by the Licensor or are sublicensed to You under the terms of this License with the permission of the contributor(s) of those copyrights and patent rights. Except as expressly stated in the immediately proceeding sentence, the Original Work is provided under this License on an "AS IS" BASIS and WITHOUT WARRANTY, either express or implied, including, without limitation, the warranties of NON-INFRINGEMENT, MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY OF THE ORIGINAL WORK IS WITH YOU. This DISCLAIMER OF WARRANTY constitutes an essential part of this License. No license to Original Work is granted hereunder except under this disclaimer. - -8) Limitation of Liability. Under no circumstances and under no legal theory, whether in tort (including negligence), contract, or otherwise, shall the Licensor be liable to any person for any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or the use of the Original Work including, without limitation, damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses. This limitation of liability shall not apply to liability for death or personal injury resulting from Licensor's negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. - -9) Acceptance and Termination. If You distribute copies of the Original Work or a Derivative Work, You must make a reasonable effort under the circumstances to obtain the express assent of recipients to the terms of this License. Nothing else but this License (or another written agreement between Licensor and You) grants You permission to create Derivative Works based upon the Original Work or to exercise any of the rights granted in Section 1 herein, and any attempt to do so except under the terms of this License (or another written agreement between Licensor and You) is expressly prohibited by U.S. copyright law, the equivalent laws of other countries, and by international treaty. Therefore, by exercising any of the rights granted to You in Section 1 herein, You indicate Your acceptance of this License and all of its terms and conditions. - -10) Termination for Patent Action. This License shall terminate automatically and You may no longer exercise any of the rights granted to You by this License as of the date You commence an action, including a cross-claim or counterclaim, against Licensor or any licensee alleging that the Original Work infringes a patent. This termination provision shall not apply for an action alleging patent infringement by combinations of the Original Work with other software or hardware. - -11) Jurisdiction, Venue and Governing Law. Any action or suit relating to this License may be brought only in the courts of a jurisdiction wherein the Licensor resides or in which Licensor conducts its primary business, and under the laws of that jurisdiction excluding its conflict-of-law provisions. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any use of the Original Work outside the scope of this License or after its termination shall be subject to the requirements and penalties of the U.S. Copyright Act, 17 U.S.C. § 101 et seq., the equivalent laws of other countries, and international treaty. This section shall survive the termination of this License. - -12) Attorneys Fees. In any action to enforce the terms of this License or seeking damages relating thereto, the prevailing party shall be entitled to recover its costs and expenses, including, without limitation, reasonable attorneys' fees and costs incurred in connection with such action, including any appeal of such action. This section shall survive the termination of this License. - -13) Miscellaneous. This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. - -14) Definition of "You" in This License. "You" throughout this License, whether in upper or lower case, means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License. For legal entities, "You" includes any entity that controls, is controlled by, or is under common control with you. For purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. - -15) Right to Use. You may use the Original Work in all ways not otherwise restricted or conditioned by this License or by law, and Licensor promises not to interfere with or be responsible for such uses by You. - -This license is Copyright (C) 2003-2004 Lawrence E. Rosen. All rights reserved. Permission is hereby granted to copy and distribute this license without modification. This license may not be modified without the express written permission of its copyright owner. diff --git a/contrib/python/simplejson/py3/README.rst b/contrib/python/simplejson/py3/README.rst deleted file mode 100644 index 6580ddac899..00000000000 --- a/contrib/python/simplejson/py3/README.rst +++ /dev/null @@ -1,34 +0,0 @@ -simplejson ----------- - -simplejson is a simple, fast, complete, correct and extensible -JSON <http://json.org> encoder and decoder for Python 3.3+ -with legacy support for Python 2.5+. It is pure Python code -with no dependencies, but includes an optional C extension -for a serious speed boost. - -The latest documentation for simplejson can be read online here: -https://simplejson.readthedocs.io/ - -simplejson is the externally maintained development version of the -json library included with Python (since 2.6). This version is tested -with the latest Python 3.8 and maintains backwards compatibility -with Python 3.3+ and the legacy Python 2.5 - Python 2.7 releases. - -The encoder can be specialized to provide serialization in any kind of -situation, without any special support by the objects to be serialized -(somewhat like pickle). This is best done with the ``default`` kwarg -to dumps. - -The decoder can handle incoming JSON strings of any specified encoding -(UTF-8 by default). It can also be specialized to post-process JSON -objects with the ``object_hook`` or ``object_pairs_hook`` kwargs. This -is particularly useful for implementing protocols such as JSON-RPC -that have a richer type system than JSON itself. - -For those of you that have legacy systems to maintain, there is a -very old fork of simplejson in the `python2.2`_ branch that supports -Python 2.2. This is based on a very old version of simplejson, -is not maintained, and should only be used as a last resort. - -.. _python2.2: https://github.com/simplejson/simplejson/tree/python2.2 diff --git a/contrib/python/simplejson/py3/simplejson/__init__.py b/contrib/python/simplejson/py3/simplejson/__init__.py deleted file mode 100644 index a526d8c7823..00000000000 --- a/contrib/python/simplejson/py3/simplejson/__init__.py +++ /dev/null @@ -1,562 +0,0 @@ -r"""JSON (JavaScript Object Notation) <http://json.org> is a subset of -JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data -interchange format. - -:mod:`simplejson` exposes an API familiar to users of the standard library -:mod:`marshal` and :mod:`pickle` modules. It is the externally maintained -version of the :mod:`json` library contained in Python 2.6, but maintains -compatibility back to Python 2.5 and (currently) has significant performance -advantages, even without using the optional C extension for speedups. - -Encoding basic Python object hierarchies:: - - >>> import simplejson as json - >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}]) - '["foo", {"bar": ["baz", null, 1.0, 2]}]' - >>> print(json.dumps("\"foo\bar")) - "\"foo\bar" - >>> print(json.dumps(u'\u1234')) - "\u1234" - >>> print(json.dumps('\\')) - "\\" - >>> print(json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True)) - {"a": 0, "b": 0, "c": 0} - >>> from simplejson.compat import StringIO - >>> io = StringIO() - >>> json.dump(['streaming API'], io) - >>> io.getvalue() - '["streaming API"]' - -Compact encoding:: - - >>> import simplejson as json - >>> obj = [1,2,3,{'4': 5, '6': 7}] - >>> json.dumps(obj, separators=(',',':'), sort_keys=True) - '[1,2,3,{"4":5,"6":7}]' - -Pretty printing:: - - >>> import simplejson as json - >>> print(json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=' ')) - { - "4": 5, - "6": 7 - } - -Decoding JSON:: - - >>> import simplejson as json - >>> obj = [u'foo', {u'bar': [u'baz', None, 1.0, 2]}] - >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj - True - >>> json.loads('"\\"foo\\bar"') == u'"foo\x08ar' - True - >>> from simplejson.compat import StringIO - >>> io = StringIO('["streaming API"]') - >>> json.load(io)[0] == 'streaming API' - True - -Specializing JSON object decoding:: - - >>> import simplejson as json - >>> def as_complex(dct): - ... if '__complex__' in dct: - ... return complex(dct['real'], dct['imag']) - ... return dct - ... - >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}', - ... object_hook=as_complex) - (1+2j) - >>> from decimal import Decimal - >>> json.loads('1.1', parse_float=Decimal) == Decimal('1.1') - True - -Specializing JSON object encoding:: - - >>> import simplejson as json - >>> def encode_complex(obj): - ... if isinstance(obj, complex): - ... return [obj.real, obj.imag] - ... raise TypeError('Object of type %s is not JSON serializable' % - ... obj.__class__.__name__) - ... - >>> json.dumps(2 + 1j, default=encode_complex) - '[2.0, 1.0]' - >>> json.JSONEncoder(default=encode_complex).encode(2 + 1j) - '[2.0, 1.0]' - >>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j)) - '[2.0, 1.0]' - -Using simplejson.tool from the shell to validate and pretty-print:: - - $ echo '{"json":"obj"}' | python -m simplejson.tool - { - "json": "obj" - } - $ echo '{ 1.2:3.4}' | python -m simplejson.tool - Expecting property name: line 1 column 3 (char 2) - -Parsing multiple documents serialized as JSON lines (newline-delimited JSON):: - - >>> import simplejson as json - >>> def loads_lines(docs): - ... for doc in docs.splitlines(): - ... yield json.loads(doc) - ... - >>> sum(doc["count"] for doc in loads_lines('{"count":1}\n{"count":2}\n{"count":3}\n')) - 6 - -Serializing multiple objects to JSON lines (newline-delimited JSON):: - - >>> import simplejson as json - >>> def dumps_lines(objs): - ... for obj in objs: - ... yield json.dumps(obj, separators=(',',':')) + '\n' - ... - >>> ''.join(dumps_lines([{'count': 1}, {'count': 2}, {'count': 3}])) - '{"count":1}\n{"count":2}\n{"count":3}\n' - -""" -from __future__ import absolute_import -__version__ = '3.20.2' -__all__ = [ - 'dump', 'dumps', 'load', 'loads', - 'JSONDecoder', 'JSONDecodeError', 'JSONEncoder', - 'OrderedDict', 'simple_first', 'RawJSON' -] - -__author__ = 'Bob Ippolito <[email protected]>' - -from decimal import Decimal - -from .errors import JSONDecodeError -from .raw_json import RawJSON -from .decoder import JSONDecoder -from .encoder import JSONEncoder, JSONEncoderForHTML -def _import_OrderedDict(): - import collections - try: - return collections.OrderedDict - except AttributeError: - from . import ordered_dict - return ordered_dict.OrderedDict -OrderedDict = _import_OrderedDict() - -def _import_c_make_encoder(): - try: - from ._speedups import make_encoder - return make_encoder - except ImportError: - return None - -_default_encoder = JSONEncoder() - -def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, - allow_nan=False, cls=None, indent=None, separators=None, - encoding='utf-8', default=None, use_decimal=True, - namedtuple_as_object=True, tuple_as_array=True, - bigint_as_string=False, sort_keys=False, item_sort_key=None, - for_json=False, ignore_nan=False, int_as_string_bitcount=None, - iterable_as_array=False, **kw): - """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a - ``.write()``-supporting file-like object). - - If *skipkeys* is true then ``dict`` keys that are not basic types - (``str``, ``int``, ``long``, ``float``, ``bool``, ``None``) - will be skipped instead of raising a ``TypeError``. - - If *ensure_ascii* is false (default: ``True``), then the output may - contain non-ASCII characters, so long as they do not need to be escaped - by JSON. When it is true, all non-ASCII characters are escaped. - - If *allow_nan* is true (default: ``False``), then out of range ``float`` - values (``nan``, ``inf``, ``-inf``) will be serialized to - their JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``) - instead of raising a ValueError. See - *ignore_nan* for ECMA-262 compliant behavior. - - If *indent* is a string, then JSON array elements and object members - will be pretty-printed with a newline followed by that string repeated - for each level of nesting. ``None`` (the default) selects the most compact - representation without any newlines. - - If specified, *separators* should be an - ``(item_separator, key_separator)`` tuple. The default is ``(', ', ': ')`` - if *indent* is ``None`` and ``(',', ': ')`` otherwise. To get the most - compact JSON representation, you should specify ``(',', ':')`` to eliminate - whitespace. - - *encoding* is the character encoding for str instances, default is UTF-8. - - *default(obj)* is a function that should return a serializable version - of obj or raise ``TypeError``. The default simply raises ``TypeError``. - - If *use_decimal* is true (default: ``True``) then decimal.Decimal - will be natively serialized to JSON with full precision. - - If *namedtuple_as_object* is true (default: ``True``), - :class:`tuple` subclasses with ``_asdict()`` methods will be encoded - as JSON objects. - - If *tuple_as_array* is true (default: ``True``), - :class:`tuple` (and subclasses) will be encoded as JSON arrays. - - If *iterable_as_array* is true (default: ``False``), - any object not in the above table that implements ``__iter__()`` - will be encoded as a JSON array. - - If *bigint_as_string* is true (default: ``False``), ints 2**53 and higher - or lower than -2**53 will be encoded as strings. This is to avoid the - rounding that happens in Javascript otherwise. Note that this is still a - lossy operation that will not round-trip correctly and should be used - sparingly. - - If *int_as_string_bitcount* is a positive number (n), then int of size - greater than or equal to 2**n or lower than or equal to -2**n will be - encoded as strings. - - If specified, *item_sort_key* is a callable used to sort the items in - each dictionary. This is useful if you want to sort items other than - in alphabetical order by key. This option takes precedence over - *sort_keys*. - - If *sort_keys* is true (default: ``False``), the output of dictionaries - will be sorted by item. - - If *for_json* is true (default: ``False``), objects with a ``for_json()`` - method will use the return value of that method for encoding as JSON - instead of the object. - - If *ignore_nan* is true (default: ``False``), then out of range - :class:`float` values (``nan``, ``inf``, ``-inf``) will be serialized as - ``null`` in compliance with the ECMA-262 specification. If true, this will - override *allow_nan*. - - To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the - ``.default()`` method to serialize additional types), specify it with - the ``cls`` kwarg. NOTE: You should use *default* or *for_json* instead - of subclassing whenever possible. - - """ - # cached encoder - if (not skipkeys and ensure_ascii and - check_circular and not allow_nan and - cls is None and indent is None and separators is None and - encoding == 'utf-8' and default is None and use_decimal - and namedtuple_as_object and tuple_as_array and not iterable_as_array - and not bigint_as_string and not sort_keys - and not item_sort_key and not for_json - and not ignore_nan and int_as_string_bitcount is None - and not kw - ): - iterable = _default_encoder.iterencode(obj) - else: - if cls is None: - cls = JSONEncoder - iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii, - check_circular=check_circular, allow_nan=allow_nan, indent=indent, - separators=separators, encoding=encoding, - default=default, use_decimal=use_decimal, - namedtuple_as_object=namedtuple_as_object, - tuple_as_array=tuple_as_array, - iterable_as_array=iterable_as_array, - bigint_as_string=bigint_as_string, - sort_keys=sort_keys, - item_sort_key=item_sort_key, - for_json=for_json, - ignore_nan=ignore_nan, - int_as_string_bitcount=int_as_string_bitcount, - **kw).iterencode(obj) - # could accelerate with writelines in some versions of Python, at - # a debuggability cost - for chunk in iterable: - fp.write(chunk) - - -def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, - allow_nan=False, cls=None, indent=None, separators=None, - encoding='utf-8', default=None, use_decimal=True, - namedtuple_as_object=True, tuple_as_array=True, - bigint_as_string=False, sort_keys=False, item_sort_key=None, - for_json=False, ignore_nan=False, int_as_string_bitcount=None, - iterable_as_array=False, **kw): - """Serialize ``obj`` to a JSON formatted ``str``. - - If ``skipkeys`` is true then ``dict`` keys that are not basic types - (``str``, ``int``, ``long``, ``float``, ``bool``, ``None``) - will be skipped instead of raising a ``TypeError``. - - If *ensure_ascii* is false (default: ``True``), then the output may - contain non-ASCII characters, so long as they do not need to be escaped - by JSON. When it is true, all non-ASCII characters are escaped. - - If ``check_circular`` is false, then the circular reference check - for container types will be skipped and a circular reference will - result in an ``OverflowError`` (or worse). - - If *allow_nan* is true (default: ``False``), then out of range ``float`` - values (``nan``, ``inf``, ``-inf``) will be serialized to - their JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``) - instead of raising a ValueError. See - *ignore_nan* for ECMA-262 compliant behavior. - - If ``indent`` is a string, then JSON array elements and object members - will be pretty-printed with a newline followed by that string repeated - for each level of nesting. ``None`` (the default) selects the most compact - representation without any newlines. For backwards compatibility with - versions of simplejson earlier than 2.1.0, an integer is also accepted - and is converted to a string with that many spaces. - - If specified, ``separators`` should be an - ``(item_separator, key_separator)`` tuple. The default is ``(', ', ': ')`` - if *indent* is ``None`` and ``(',', ': ')`` otherwise. To get the most - compact JSON representation, you should specify ``(',', ':')`` to eliminate - whitespace. - - ``encoding`` is the character encoding for bytes instances, default is - UTF-8. - - ``default(obj)`` is a function that should return a serializable version - of obj or raise TypeError. The default simply raises TypeError. - - If *use_decimal* is true (default: ``True``) then decimal.Decimal - will be natively serialized to JSON with full precision. - - If *namedtuple_as_object* is true (default: ``True``), - :class:`tuple` subclasses with ``_asdict()`` methods will be encoded - as JSON objects. - - If *tuple_as_array* is true (default: ``True``), - :class:`tuple` (and subclasses) will be encoded as JSON arrays. - - If *iterable_as_array* is true (default: ``False``), - any object not in the above table that implements ``__iter__()`` - will be encoded as a JSON array. - - If *bigint_as_string* is true (not the default), ints 2**53 and higher - or lower than -2**53 will be encoded as strings. This is to avoid the - rounding that happens in Javascript otherwise. - - If *int_as_string_bitcount* is a positive number (n), then int of size - greater than or equal to 2**n or lower than or equal to -2**n will be - encoded as strings. - - If specified, *item_sort_key* is a callable used to sort the items in - each dictionary. This is useful if you want to sort items other than - in alphabetical order by key. This option takes precedence over - *sort_keys*. - - If *sort_keys* is true (default: ``False``), the output of dictionaries - will be sorted by item. - - If *for_json* is true (default: ``False``), objects with a ``for_json()`` - method will use the return value of that method for encoding as JSON - instead of the object. - - If *ignore_nan* is true (default: ``False``), then out of range - :class:`float` values (``nan``, ``inf``, ``-inf``) will be serialized as - ``null`` in compliance with the ECMA-262 specification. If true, this will - override *allow_nan*. - - To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the - ``.default()`` method to serialize additional types), specify it with - the ``cls`` kwarg. NOTE: You should use *default* instead of subclassing - whenever possible. - - """ - # cached encoder - if (not skipkeys and ensure_ascii and - check_circular and not allow_nan and - cls is None and indent is None and separators is None and - encoding == 'utf-8' and default is None and use_decimal - and namedtuple_as_object and tuple_as_array and not iterable_as_array - and not bigint_as_string and not sort_keys - and not item_sort_key and not for_json - and not ignore_nan and int_as_string_bitcount is None - and not kw - ): - return _default_encoder.encode(obj) - if cls is None: - cls = JSONEncoder - return cls( - skipkeys=skipkeys, ensure_ascii=ensure_ascii, - check_circular=check_circular, allow_nan=allow_nan, indent=indent, - separators=separators, encoding=encoding, default=default, - use_decimal=use_decimal, - namedtuple_as_object=namedtuple_as_object, - tuple_as_array=tuple_as_array, - iterable_as_array=iterable_as_array, - bigint_as_string=bigint_as_string, - sort_keys=sort_keys, - item_sort_key=item_sort_key, - for_json=for_json, - ignore_nan=ignore_nan, - int_as_string_bitcount=int_as_string_bitcount, - **kw).encode(obj) - - -_default_decoder = JSONDecoder() - - -def load(fp, encoding=None, cls=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, object_pairs_hook=None, - use_decimal=False, allow_nan=False, **kw): - """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing - a JSON document as `str` or `bytes`) to a Python object. - - *encoding* determines the encoding used to interpret any - `bytes` objects decoded by this instance (``'utf-8'`` by - default). It has no effect when decoding `str` objects. - - *object_hook*, if specified, will be called with the result of every - JSON object decoded and its return value will be used in place of the - given :class:`dict`. This can be used to provide custom - deserializations (e.g. to support JSON-RPC class hinting). - - *object_pairs_hook* is an optional function that will be called with - the result of any object literal decode with an ordered list of pairs. - The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders - that rely on the order that the key and value pairs are decoded (for - example, :func:`collections.OrderedDict` will remember the order of - insertion). If *object_hook* is also defined, the *object_pairs_hook* - takes priority. - - *parse_float*, if specified, will be called with the string of every - JSON float to be decoded. By default, this is equivalent to - ``float(num_str)``. This can be used to use another datatype or parser - for JSON floats (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every - JSON int to be decoded. By default, this is equivalent to - ``int(num_str)``. This can be used to use another datatype or parser - for JSON integers (e.g. :class:`float`). - - *allow_nan*, if True (default false), will allow the parser to - accept the non-standard floats ``NaN``, ``Infinity``, and ``-Infinity`` - and enable the use of the deprecated *parse_constant*. - - If *use_decimal* is true (default: ``False``) then it implies - parse_float=decimal.Decimal for parity with ``dump``. - - *parse_constant*, if specified, will be - called with one of the following strings: ``'-Infinity'``, - ``'Infinity'``, ``'NaN'``. It is not recommended to use this feature, - as it is rare to parse non-compliant JSON containing these values. - - To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` - kwarg. NOTE: You should use *object_hook* or *object_pairs_hook* instead - of subclassing whenever possible. - - """ - return loads(fp.read(), - encoding=encoding, cls=cls, object_hook=object_hook, - parse_float=parse_float, parse_int=parse_int, - parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, - use_decimal=use_decimal, allow_nan=allow_nan, **kw) - - -def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, object_pairs_hook=None, - use_decimal=False, allow_nan=False, **kw): - """Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON - document) to a Python object. - - *encoding* determines the encoding used to interpret any - :class:`bytes` objects decoded by this instance (``'utf-8'`` by - default). It has no effect when decoding :class:`unicode` objects. - - *object_hook*, if specified, will be called with the result of every - JSON object decoded and its return value will be used in place of the - given :class:`dict`. This can be used to provide custom - deserializations (e.g. to support JSON-RPC class hinting). - - *object_pairs_hook* is an optional function that will be called with - the result of any object literal decode with an ordered list of pairs. - The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders - that rely on the order that the key and value pairs are decoded (for - example, :func:`collections.OrderedDict` will remember the order of - insertion). If *object_hook* is also defined, the *object_pairs_hook* - takes priority. - - *parse_float*, if specified, will be called with the string of every - JSON float to be decoded. By default, this is equivalent to - ``float(num_str)``. This can be used to use another datatype or parser - for JSON floats (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every - JSON int to be decoded. By default, this is equivalent to - ``int(num_str)``. This can be used to use another datatype or parser - for JSON integers (e.g. :class:`float`). - - *allow_nan*, if True (default false), will allow the parser to - accept the non-standard floats ``NaN``, ``Infinity``, and ``-Infinity`` - and enable the use of the deprecated *parse_constant*. - - If *use_decimal* is true (default: ``False``) then it implies - parse_float=decimal.Decimal for parity with ``dump``. - - *parse_constant*, if specified, will be - called with one of the following strings: ``'-Infinity'``, - ``'Infinity'``, ``'NaN'``. It is not recommended to use this feature, - as it is rare to parse non-compliant JSON containing these values. - - To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` - kwarg. NOTE: You should use *object_hook* or *object_pairs_hook* instead - of subclassing whenever possible. - - """ - if (cls is None and encoding is None and object_hook is None and - parse_int is None and parse_float is None and - parse_constant is None and object_pairs_hook is None - and not use_decimal and not allow_nan and not kw): - return _default_decoder.decode(s) - if cls is None: - cls = JSONDecoder - if object_hook is not None: - kw['object_hook'] = object_hook - if object_pairs_hook is not None: - kw['object_pairs_hook'] = object_pairs_hook - if parse_float is not None: - kw['parse_float'] = parse_float - if parse_int is not None: - kw['parse_int'] = parse_int - if parse_constant is not None: - kw['parse_constant'] = parse_constant - if use_decimal: - if parse_float is not None: - raise TypeError("use_decimal=True implies parse_float=Decimal") - kw['parse_float'] = Decimal - if allow_nan: - kw['allow_nan'] = True - return cls(encoding=encoding, **kw).decode(s) - - -def _toggle_speedups(enabled): - from . import decoder as dec - from . import encoder as enc - from . import scanner as scan - c_make_encoder = _import_c_make_encoder() - if enabled: - dec.scanstring = dec.c_scanstring or dec.py_scanstring - enc.c_make_encoder = c_make_encoder - enc.encode_basestring_ascii = (enc.c_encode_basestring_ascii or - enc.py_encode_basestring_ascii) - scan.make_scanner = scan.c_make_scanner or scan.py_make_scanner - else: - dec.scanstring = dec.py_scanstring - enc.c_make_encoder = None - enc.encode_basestring_ascii = enc.py_encode_basestring_ascii - scan.make_scanner = scan.py_make_scanner - dec.make_scanner = scan.make_scanner - global _default_decoder - _default_decoder = JSONDecoder() - global _default_encoder - _default_encoder = JSONEncoder() - -def simple_first(kv): - """Helper function to pass to item_sort_key to sort simple - elements to the top, then container elements. - """ - return (isinstance(kv[1], (list, dict, tuple)), kv[0]) diff --git a/contrib/python/simplejson/py3/simplejson/_speedups.c b/contrib/python/simplejson/py3/simplejson/_speedups.c deleted file mode 100644 index 4133b9bc761..00000000000 --- a/contrib/python/simplejson/py3/simplejson/_speedups.c +++ /dev/null @@ -1,3414 +0,0 @@ -/* -*- mode: C; c-file-style: "python"; c-basic-offset: 4 -*- */ -#include "Python.h" -#include "structmember.h" - -#if PY_MAJOR_VERSION >= 3 -#define PyInt_FromSsize_t PyLong_FromSsize_t -#define PyInt_AsSsize_t PyLong_AsSsize_t -#define PyInt_Check(obj) 0 -#define PyInt_CheckExact(obj) 0 -#define JSON_UNICHR Py_UCS4 -#define JSON_InternFromString PyUnicode_InternFromString -#define PyString_GET_SIZE PyUnicode_GET_LENGTH -#define JSON_StringCheck PyUnicode_Check -#define PY2_UNUSED -#define PY3_UNUSED UNUSED -#else /* PY_MAJOR_VERSION >= 3 */ -#define PY2_UNUSED UNUSED -#define PY3_UNUSED -#define JSON_StringCheck(obj) (PyString_Check(obj) || PyUnicode_Check(obj)) -#define PyBytes_Check PyString_Check -#define PyUnicode_READY(obj) 0 -#define PyUnicode_KIND(obj) (sizeof(Py_UNICODE)) -#define PyUnicode_DATA(obj) ((void *)(PyUnicode_AS_UNICODE(obj))) -#define PyUnicode_READ(kind, data, index) ((JSON_UNICHR)((const Py_UNICODE *)(data))[(index)]) -#define PyUnicode_GET_LENGTH PyUnicode_GET_SIZE -#define JSON_UNICHR Py_UNICODE -#define JSON_InternFromString PyString_InternFromString -#endif /* PY_MAJOR_VERSION < 3 */ - -#if PY_VERSION_HEX < 0x03090000 -#if !defined(PyObject_CallNoArgs) -#define PyObject_CallNoArgs(callable) PyObject_CallFunctionObjArgs(callable, NULL); -#endif -#if !defined(PyObject_CallOneArg) -#define PyObject_CallOneArg(callable, arg) PyObject_CallFunctionObjArgs(callable, arg, NULL); -#endif -#endif /* PY_VERSION_HEX < 0x03090000 */ - -#if PY_VERSION_HEX < 0x02070000 -#if !defined(PyOS_string_to_double) -#define PyOS_string_to_double json_PyOS_string_to_double -static double -json_PyOS_string_to_double(const char *s, char **endptr, PyObject *overflow_exception); -static double -json_PyOS_string_to_double(const char *s, char **endptr, PyObject *overflow_exception) -{ - double x; - assert(endptr == NULL); - assert(overflow_exception == NULL); - PyFPE_START_PROTECT("json_PyOS_string_to_double", return -1.0;) - x = PyOS_ascii_atof(s); - PyFPE_END_PROTECT(x) - return x; -} -#endif -#endif /* PY_VERSION_HEX < 0x02070000 */ - -#if PY_VERSION_HEX < 0x02060000 -#if !defined(Py_TYPE) -#define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) -#endif -#if !defined(Py_SIZE) -#define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) -#endif -#if !defined(PyVarObject_HEAD_INIT) -#define PyVarObject_HEAD_INIT(type, size) PyObject_HEAD_INIT(type) size, -#endif -#endif /* PY_VERSION_HEX < 0x02060000 */ - -#ifdef __GNUC__ -#define UNUSED __attribute__((__unused__)) -#else -#define UNUSED -#endif - -#define DEFAULT_ENCODING "utf-8" - -#define PyScanner_Check(op) PyObject_TypeCheck(op, &PyScannerType) -#define PyScanner_CheckExact(op) (Py_TYPE(op) == &PyScannerType) -#define PyEncoder_Check(op) PyObject_TypeCheck(op, &PyEncoderType) -#define PyEncoder_CheckExact(op) (Py_TYPE(op) == &PyEncoderType) - -#define JSON_ALLOW_NAN 1 -#define JSON_IGNORE_NAN 2 - -static PyObject *JSON_Infinity = NULL; -static PyObject *JSON_NegInfinity = NULL; -static PyObject *JSON_NaN = NULL; -static PyObject *JSON_EmptyUnicode = NULL; -#if PY_MAJOR_VERSION < 3 -static PyObject *JSON_EmptyStr = NULL; -#endif - -static PyTypeObject PyScannerType; -static PyTypeObject PyEncoderType; - -typedef struct { - PyObject *large_strings; /* A list of previously accumulated large strings */ - PyObject *small_strings; /* Pending small strings */ -} JSON_Accu; - -static int -JSON_Accu_Init(JSON_Accu *acc); -static int -JSON_Accu_Accumulate(JSON_Accu *acc, PyObject *unicode); -static PyObject * -JSON_Accu_FinishAsList(JSON_Accu *acc); -static void -JSON_Accu_Destroy(JSON_Accu *acc); - -#define ERR_EXPECTING_VALUE "Expecting value" -#define ERR_ARRAY_DELIMITER "Expecting ',' delimiter or ']'" -#define ERR_ARRAY_VALUE_FIRST "Expecting value or ']'" -#define ERR_OBJECT_DELIMITER "Expecting ',' delimiter or '}'" -#define ERR_OBJECT_PROPERTY "Expecting property name enclosed in double quotes" -#define ERR_OBJECT_PROPERTY_FIRST "Expecting property name enclosed in double quotes or '}'" -#define ERR_OBJECT_PROPERTY_DELIMITER "Expecting ':' delimiter" -#define ERR_STRING_UNTERMINATED "Unterminated string starting at" -#define ERR_STRING_CONTROL "Invalid control character %r at" -#define ERR_STRING_ESC1 "Invalid \\X escape sequence %r" -#define ERR_STRING_ESC4 "Invalid \\uXXXX escape sequence" -#define FOR_JSON_METHOD_NAME "for_json" -#define ASDICT_METHOD_NAME "_asdict" - - -typedef struct _PyScannerObject { - PyObject_HEAD - PyObject *encoding; - PyObject *strict_bool; - int strict; - PyObject *object_hook; - PyObject *pairs_hook; - PyObject *parse_float; - PyObject *parse_int; - PyObject *parse_constant; - PyObject *memo; -} PyScannerObject; - -static PyMemberDef scanner_members[] = { - {"encoding", T_OBJECT, offsetof(PyScannerObject, encoding), READONLY, "encoding"}, - {"strict", T_OBJECT, offsetof(PyScannerObject, strict_bool), READONLY, "strict"}, - {"object_hook", T_OBJECT, offsetof(PyScannerObject, object_hook), READONLY, "object_hook"}, - {"object_pairs_hook", T_OBJECT, offsetof(PyScannerObject, pairs_hook), READONLY, "object_pairs_hook"}, - {"parse_float", T_OBJECT, offsetof(PyScannerObject, parse_float), READONLY, "parse_float"}, - {"parse_int", T_OBJECT, offsetof(PyScannerObject, parse_int), READONLY, "parse_int"}, - {"parse_constant", T_OBJECT, offsetof(PyScannerObject, parse_constant), READONLY, "parse_constant"}, - {NULL} -}; - -typedef struct _PyEncoderObject { - PyObject_HEAD - PyObject *markers; - PyObject *defaultfn; - PyObject *encoder; - PyObject *indent; - PyObject *key_separator; - PyObject *item_separator; - PyObject *sort_keys; - PyObject *key_memo; - PyObject *encoding; - PyObject *Decimal; - PyObject *skipkeys_bool; - int skipkeys; - int fast_encode; - /* 0, JSON_ALLOW_NAN, JSON_IGNORE_NAN */ - int allow_or_ignore_nan; - int use_decimal; - int namedtuple_as_object; - int tuple_as_array; - int iterable_as_array; - PyObject *max_long_size; - PyObject *min_long_size; - PyObject *item_sort_key; - PyObject *item_sort_kw; - int for_json; -} PyEncoderObject; - -static PyMemberDef encoder_members[] = { - {"markers", T_OBJECT, offsetof(PyEncoderObject, markers), READONLY, "markers"}, - {"default", T_OBJECT, offsetof(PyEncoderObject, defaultfn), READONLY, "default"}, - {"encoder", T_OBJECT, offsetof(PyEncoderObject, encoder), READONLY, "encoder"}, - {"encoding", T_OBJECT, offsetof(PyEncoderObject, encoder), READONLY, "encoding"}, - {"indent", T_OBJECT, offsetof(PyEncoderObject, indent), READONLY, "indent"}, - {"key_separator", T_OBJECT, offsetof(PyEncoderObject, key_separator), READONLY, "key_separator"}, - {"item_separator", T_OBJECT, offsetof(PyEncoderObject, item_separator), READONLY, "item_separator"}, - {"sort_keys", T_OBJECT, offsetof(PyEncoderObject, sort_keys), READONLY, "sort_keys"}, - /* Python 2.5 does not support T_BOOl */ - {"skipkeys", T_OBJECT, offsetof(PyEncoderObject, skipkeys_bool), READONLY, "skipkeys"}, - {"key_memo", T_OBJECT, offsetof(PyEncoderObject, key_memo), READONLY, "key_memo"}, - {"item_sort_key", T_OBJECT, offsetof(PyEncoderObject, item_sort_key), READONLY, "item_sort_key"}, - {"max_long_size", T_OBJECT, offsetof(PyEncoderObject, max_long_size), READONLY, "max_long_size"}, - {"min_long_size", T_OBJECT, offsetof(PyEncoderObject, min_long_size), READONLY, "min_long_size"}, - {NULL} -}; - -static PyObject * -join_list_unicode(PyObject *lst); -static PyObject * -JSON_ParseEncoding(PyObject *encoding); -static PyObject * -maybe_quote_bigint(PyEncoderObject* s, PyObject *encoded, PyObject *obj); -static Py_ssize_t -ascii_char_size(JSON_UNICHR c); -static Py_ssize_t -ascii_escape_char(JSON_UNICHR c, char *output, Py_ssize_t chars); -static PyObject * -ascii_escape_unicode(PyObject *pystr); -static PyObject * -ascii_escape_str(PyObject *pystr); -static PyObject * -py_encode_basestring_ascii(PyObject* self UNUSED, PyObject *pystr); -#if PY_MAJOR_VERSION < 3 -static PyObject * -join_list_string(PyObject *lst); -static PyObject * -scan_once_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr); -static PyObject * -scanstring_str(PyObject *pystr, Py_ssize_t end, char *encoding, int strict, Py_ssize_t *next_end_ptr); -static PyObject * -_parse_object_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr); -#endif -static PyObject * -scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next_end_ptr); -static PyObject * -scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr); -static PyObject * -_build_rval_index_tuple(PyObject *rval, Py_ssize_t idx); -static PyObject * -scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds); -static void -scanner_dealloc(PyObject *self); -static int -scanner_clear(PyObject *self); -static PyObject * -encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds); -static void -encoder_dealloc(PyObject *self); -static int -encoder_clear(PyObject *self); -static int -is_raw_json(PyObject *obj); -static PyObject * -encoder_stringify_key(PyEncoderObject *s, PyObject *key); -static int -encoder_listencode_list(PyEncoderObject *s, JSON_Accu *rval, PyObject *seq, Py_ssize_t indent_level); -static int -encoder_listencode_obj(PyEncoderObject *s, JSON_Accu *rval, PyObject *obj, Py_ssize_t indent_level); -static int -encoder_listencode_dict(PyEncoderObject *s, JSON_Accu *rval, PyObject *dct, Py_ssize_t indent_level); -static PyObject * -_encoded_const(PyObject *obj); -static void -raise_errmsg(char *msg, PyObject *s, Py_ssize_t end); -static PyObject * -encoder_encode_string(PyEncoderObject *s, PyObject *obj); -static int -_convertPyInt_AsSsize_t(PyObject *o, Py_ssize_t *size_ptr); -static PyObject * -_convertPyInt_FromSsize_t(Py_ssize_t *size_ptr); -static int -_call_json_method(PyObject *obj, const char *method_name, PyObject **result); -static PyObject * -encoder_encode_float(PyEncoderObject *s, PyObject *obj); -static PyObject * -moduleinit(void); - -#define S_CHAR(c) (c >= ' ' && c <= '~' && c != '\\' && c != '"') -#define IS_WHITESPACE(c) (((c) == ' ') || ((c) == '\t') || ((c) == '\n') || ((c) == '\r')) - -#define MIN_EXPANSION 6 - -static PyObject* RawJSONType = NULL; -static int -is_raw_json(PyObject *obj) -{ - return PyObject_IsInstance(obj, RawJSONType) ? 1 : 0; -} - -static int -JSON_Accu_Init(JSON_Accu *acc) -{ - /* Lazily allocated */ - acc->large_strings = NULL; - acc->small_strings = PyList_New(0); - if (acc->small_strings == NULL) - return -1; - return 0; -} - -static int -flush_accumulator(JSON_Accu *acc) -{ - Py_ssize_t nsmall = PyList_GET_SIZE(acc->small_strings); - if (nsmall) { - int ret; - PyObject *joined; - if (acc->large_strings == NULL) { - acc->large_strings = PyList_New(0); - if (acc->large_strings == NULL) - return -1; - } -#if PY_MAJOR_VERSION >= 3 - joined = join_list_unicode(acc->small_strings); -#else /* PY_MAJOR_VERSION >= 3 */ - joined = join_list_string(acc->small_strings); -#endif /* PY_MAJOR_VERSION < 3 */ - if (joined == NULL) - return -1; - if (PyList_SetSlice(acc->small_strings, 0, nsmall, NULL)) { - Py_DECREF(joined); - return -1; - } - ret = PyList_Append(acc->large_strings, joined); - Py_DECREF(joined); - return ret; - } - return 0; -} - -static int -JSON_Accu_Accumulate(JSON_Accu *acc, PyObject *unicode) -{ - Py_ssize_t nsmall; -#if PY_MAJOR_VERSION >= 3 - assert(PyUnicode_Check(unicode)); -#else /* PY_MAJOR_VERSION >= 3 */ - assert(PyString_Check(unicode) || PyUnicode_Check(unicode)); -#endif /* PY_MAJOR_VERSION < 3 */ - - if (PyList_Append(acc->small_strings, unicode)) - return -1; - nsmall = PyList_GET_SIZE(acc->small_strings); - /* Each item in a list of unicode objects has an overhead (in 64-bit - * builds) of: - * - 8 bytes for the list slot - * - 56 bytes for the header of the unicode object - * that is, 64 bytes. 100000 such objects waste more than 6MB - * compared to a single concatenated string. - */ - if (nsmall < 100000) - return 0; - return flush_accumulator(acc); -} - -static PyObject * -JSON_Accu_FinishAsList(JSON_Accu *acc) -{ - int ret; - PyObject *res; - - ret = flush_accumulator(acc); - Py_CLEAR(acc->small_strings); - if (ret) { - Py_CLEAR(acc->large_strings); - return NULL; - } - res = acc->large_strings; - acc->large_strings = NULL; - if (res == NULL) - return PyList_New(0); - return res; -} - -static void -JSON_Accu_Destroy(JSON_Accu *acc) -{ - Py_CLEAR(acc->small_strings); - Py_CLEAR(acc->large_strings); -} - -static int -IS_DIGIT(JSON_UNICHR c) -{ - return c >= '0' && c <= '9'; -} - -static PyObject * -maybe_quote_bigint(PyEncoderObject* s, PyObject *encoded, PyObject *obj) -{ - if (s->max_long_size != Py_None && s->min_long_size != Py_None) { - if (PyObject_RichCompareBool(obj, s->max_long_size, Py_GE) || - PyObject_RichCompareBool(obj, s->min_long_size, Py_LE)) { -#if PY_MAJOR_VERSION >= 3 - PyObject* quoted = PyUnicode_FromFormat("\"%U\"", encoded); -#else - PyObject* quoted = PyString_FromFormat("\"%s\"", - PyString_AsString(encoded)); -#endif - Py_DECREF(encoded); - encoded = quoted; - } - } - - return encoded; -} - -static int -_call_json_method(PyObject *obj, const char *method_name, PyObject **result) -{ - int rval = 0; - PyObject *method = PyObject_GetAttrString(obj, method_name); - if (method == NULL) { - PyErr_Clear(); - return 0; - } - if (PyCallable_Check(method)) { - PyObject *tmp = PyObject_CallNoArgs(method); - if (tmp == NULL && PyErr_ExceptionMatches(PyExc_TypeError)) { - PyErr_Clear(); - } else { - // This will set result to NULL if a TypeError occurred, - // which must be checked by the caller - *result = tmp; - rval = 1; - } - } - Py_DECREF(method); - return rval; -} - -static int -_convertPyInt_AsSsize_t(PyObject *o, Py_ssize_t *size_ptr) -{ - /* PyObject to Py_ssize_t converter */ - *size_ptr = PyInt_AsSsize_t(o); - if (*size_ptr == -1 && PyErr_Occurred()) - return 0; - return 1; -} - -static PyObject * -_convertPyInt_FromSsize_t(Py_ssize_t *size_ptr) -{ - /* Py_ssize_t to PyObject converter */ - return PyInt_FromSsize_t(*size_ptr); -} - -static Py_ssize_t -ascii_escape_char(JSON_UNICHR c, char *output, Py_ssize_t chars) -{ - /* Escape unicode code point c to ASCII escape sequences - in char *output. output must have at least 12 bytes unused to - accommodate an escaped surrogate pair "\uXXXX\uXXXX" */ - if (S_CHAR(c)) { - output[chars++] = (char)c; - } - else { - output[chars++] = '\\'; - switch (c) { - case '\\': output[chars++] = (char)c; break; - case '"': output[chars++] = (char)c; break; - case '\b': output[chars++] = 'b'; break; - case '\f': output[chars++] = 'f'; break; - case '\n': output[chars++] = 'n'; break; - case '\r': output[chars++] = 'r'; break; - case '\t': output[chars++] = 't'; break; - default: -#if PY_MAJOR_VERSION >= 3 || defined(Py_UNICODE_WIDE) - if (c >= 0x10000) { - /* UTF-16 surrogate pair */ - JSON_UNICHR v = c - 0x10000; - c = 0xd800 | ((v >> 10) & 0x3ff); - output[chars++] = 'u'; - output[chars++] = "0123456789abcdef"[(c >> 12) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 8) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 4) & 0xf]; - output[chars++] = "0123456789abcdef"[(c ) & 0xf]; - c = 0xdc00 | (v & 0x3ff); - output[chars++] = '\\'; - } -#endif - output[chars++] = 'u'; - output[chars++] = "0123456789abcdef"[(c >> 12) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 8) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 4) & 0xf]; - output[chars++] = "0123456789abcdef"[(c ) & 0xf]; - } - } - return chars; -} - -static Py_ssize_t -ascii_char_size(JSON_UNICHR c) -{ - if (S_CHAR(c)) { - return 1; - } - else if (c == '\\' || - c == '"' || - c == '\b' || - c == '\f' || - c == '\n' || - c == '\r' || - c == '\t') { - return 2; - } -#if PY_MAJOR_VERSION >= 3 || defined(Py_UNICODE_WIDE) - else if (c >= 0x10000U) { - return 2 * MIN_EXPANSION; - } -#endif - else { - return MIN_EXPANSION; - } -} - -static PyObject * -ascii_escape_unicode(PyObject *pystr) -{ - /* Take a PyUnicode pystr and return a new ASCII-only escaped PyString */ - Py_ssize_t i; - Py_ssize_t input_chars = PyUnicode_GET_LENGTH(pystr); - Py_ssize_t output_size = 2; - Py_ssize_t chars; - PY2_UNUSED int kind = PyUnicode_KIND(pystr); - void *data = PyUnicode_DATA(pystr); - PyObject *rval; - char *output; - - output_size = 2; - for (i = 0; i < input_chars; i++) { - output_size += ascii_char_size(PyUnicode_READ(kind, data, i)); - } -#if PY_MAJOR_VERSION >= 3 - rval = PyUnicode_New(output_size, 127); - if (rval == NULL) { - return NULL; - } - assert(PyUnicode_KIND(rval) == PyUnicode_1BYTE_KIND); - output = (char *)PyUnicode_DATA(rval); -#else - rval = PyString_FromStringAndSize(NULL, output_size); - if (rval == NULL) { - return NULL; - } - output = PyString_AS_STRING(rval); -#endif - chars = 0; - output[chars++] = '"'; - for (i = 0; i < input_chars; i++) { - chars = ascii_escape_char(PyUnicode_READ(kind, data, i), output, chars); - } - output[chars++] = '"'; - assert(chars == output_size); - return rval; -} - -#if PY_MAJOR_VERSION >= 3 - -static PyObject * -ascii_escape_str(PyObject *pystr) -{ - PyObject *rval; - PyObject *input = PyUnicode_DecodeUTF8(PyBytes_AS_STRING(pystr), PyBytes_GET_SIZE(pystr), NULL); - if (input == NULL) - return NULL; - rval = ascii_escape_unicode(input); - Py_DECREF(input); - return rval; -} - -#else /* PY_MAJOR_VERSION >= 3 */ - -static PyObject * -ascii_escape_str(PyObject *pystr) -{ - /* Take a PyString pystr and return a new ASCII-only escaped PyString */ - Py_ssize_t i; - Py_ssize_t input_chars; - Py_ssize_t output_size; - Py_ssize_t chars; - PyObject *rval; - char *output; - char *input_str; - - input_chars = PyString_GET_SIZE(pystr); - input_str = PyString_AS_STRING(pystr); - output_size = 2; - - /* Fast path for a string that's already ASCII */ - for (i = 0; i < input_chars; i++) { - JSON_UNICHR c = (JSON_UNICHR)input_str[i]; - if (c > 0x7f) { - /* We hit a non-ASCII character, bail to unicode mode */ - PyObject *uni; - uni = PyUnicode_DecodeUTF8(input_str, input_chars, "strict"); - if (uni == NULL) { - return NULL; - } - rval = ascii_escape_unicode(uni); - Py_DECREF(uni); - return rval; - } - output_size += ascii_char_size(c); - } - - rval = PyString_FromStringAndSize(NULL, output_size); - if (rval == NULL) { - return NULL; - } - chars = 0; - output = PyString_AS_STRING(rval); - output[chars++] = '"'; - for (i = 0; i < input_chars; i++) { - chars = ascii_escape_char((JSON_UNICHR)input_str[i], output, chars); - } - output[chars++] = '"'; - assert(chars == output_size); - return rval; -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -encoder_stringify_key(PyEncoderObject *s, PyObject *key) -{ - if (PyUnicode_Check(key)) { - Py_INCREF(key); - return key; - } -#if PY_MAJOR_VERSION >= 3 - else if (PyBytes_Check(key) && s->encoding != NULL) { - const char *encoding = PyUnicode_AsUTF8(s->encoding); - if (encoding == NULL) - return NULL; - return PyUnicode_Decode( - PyBytes_AS_STRING(key), - PyBytes_GET_SIZE(key), - encoding, - NULL); - } -#else /* PY_MAJOR_VERSION >= 3 */ - else if (PyString_Check(key)) { - Py_INCREF(key); - return key; - } -#endif /* PY_MAJOR_VERSION < 3 */ - else if (PyFloat_Check(key)) { - return encoder_encode_float(s, key); - } - else if (key == Py_True || key == Py_False || key == Py_None) { - /* This must come before the PyInt_Check because - True and False are also 1 and 0.*/ - return _encoded_const(key); - } - else if (PyInt_Check(key) || PyLong_Check(key)) { - if (!(PyInt_CheckExact(key) || PyLong_CheckExact(key))) { - /* See #118, do not trust custom str/repr */ - PyObject *res; - PyObject *tmp = PyObject_CallOneArg((PyObject *)&PyLong_Type, key); - if (tmp == NULL) { - return NULL; - } - res = PyObject_Str(tmp); - Py_DECREF(tmp); - return res; - } - else { - return PyObject_Str(key); - } - } - else if (s->use_decimal && PyObject_TypeCheck(key, (PyTypeObject *)s->Decimal)) { - return PyObject_Str(key); - } - if (s->skipkeys) { - Py_INCREF(Py_None); - return Py_None; - } - PyErr_Format(PyExc_TypeError, - "keys must be str, int, float, bool or None, " - "not %.100s", key->ob_type->tp_name); - return NULL; -} - -static PyObject * -encoder_dict_iteritems(PyEncoderObject *s, PyObject *dct) -{ - PyObject *items; - PyObject *iter = NULL; - PyObject *lst = NULL; - PyObject *item = NULL; - PyObject *kstr = NULL; - PyObject *sortfun = NULL; - PyObject *sortres; - static PyObject *sortargs = NULL; - - if (sortargs == NULL) { - sortargs = PyTuple_New(0); - if (sortargs == NULL) - return NULL; - } - - if (PyDict_CheckExact(dct)) - items = PyDict_Items(dct); - else - items = PyMapping_Items(dct); - if (items == NULL) - return NULL; - iter = PyObject_GetIter(items); - Py_DECREF(items); - if (iter == NULL) - return NULL; - if (s->item_sort_kw == Py_None) - return iter; - lst = PyList_New(0); - if (lst == NULL) - goto bail; - while ((item = PyIter_Next(iter))) { - PyObject *key, *value; - if (!PyTuple_Check(item) || Py_SIZE(item) != 2) { - PyErr_SetString(PyExc_ValueError, "items must return 2-tuples"); - goto bail; - } - key = PyTuple_GET_ITEM(item, 0); - if (key == NULL) - goto bail; -#if PY_MAJOR_VERSION < 3 - else if (PyString_Check(key)) { - /* item can be added as-is */ - } -#endif /* PY_MAJOR_VERSION < 3 */ - else if (PyUnicode_Check(key)) { - /* item can be added as-is */ - } - else { - PyObject *tpl; - kstr = encoder_stringify_key(s, key); - if (kstr == NULL) - goto bail; - else if (kstr == Py_None) { - /* skipkeys */ - Py_DECREF(kstr); - continue; - } - value = PyTuple_GET_ITEM(item, 1); - if (value == NULL) - goto bail; - tpl = PyTuple_Pack(2, kstr, value); - if (tpl == NULL) - goto bail; - Py_CLEAR(kstr); - Py_DECREF(item); - item = tpl; - } - if (PyList_Append(lst, item)) - goto bail; - Py_DECREF(item); - } - Py_CLEAR(iter); - if (PyErr_Occurred()) - goto bail; - sortfun = PyObject_GetAttrString(lst, "sort"); - if (sortfun == NULL) - goto bail; - sortres = PyObject_Call(sortfun, sortargs, s->item_sort_kw); - if (!sortres) - goto bail; - Py_DECREF(sortres); - Py_CLEAR(sortfun); - iter = PyObject_GetIter(lst); - Py_CLEAR(lst); - return iter; -bail: - Py_XDECREF(sortfun); - Py_XDECREF(kstr); - Py_XDECREF(item); - Py_XDECREF(lst); - Py_XDECREF(iter); - return NULL; -} - -/* Use JSONDecodeError exception to raise a nice looking ValueError subclass */ -static PyObject *JSONDecodeError = NULL; -static void -raise_errmsg(char *msg, PyObject *s, Py_ssize_t end) -{ - PyObject *exc = PyObject_CallFunction(JSONDecodeError, "(zOO&)", msg, s, _convertPyInt_FromSsize_t, &end); - if (exc) { - PyErr_SetObject(JSONDecodeError, exc); - Py_DECREF(exc); - } -} - -static PyObject * -join_list_unicode(PyObject *lst) -{ - /* return u''.join(lst) */ - return PyUnicode_Join(JSON_EmptyUnicode, lst); -} - -#if PY_MAJOR_VERSION >= 3 -#define join_list_string join_list_unicode -#else /* PY_MAJOR_VERSION >= 3 */ -static PyObject * -join_list_string(PyObject *lst) -{ - /* return ''.join(lst) */ - static PyObject *joinfn = NULL; - if (joinfn == NULL) { - joinfn = PyObject_GetAttrString(JSON_EmptyStr, "join"); - if (joinfn == NULL) - return NULL; - } - return PyObject_CallOneArg(joinfn, lst); -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -_build_rval_index_tuple(PyObject *rval, Py_ssize_t idx) -{ - /* return (rval, idx) tuple, stealing reference to rval */ - PyObject *tpl; - PyObject *pyidx; - /* - steal a reference to rval, returns (rval, idx) - */ - if (rval == NULL) { - assert(PyErr_Occurred()); - return NULL; - } - pyidx = PyInt_FromSsize_t(idx); - if (pyidx == NULL) { - Py_DECREF(rval); - return NULL; - } - tpl = PyTuple_New(2); - if (tpl == NULL) { - Py_DECREF(pyidx); - Py_DECREF(rval); - return NULL; - } - PyTuple_SET_ITEM(tpl, 0, rval); - PyTuple_SET_ITEM(tpl, 1, pyidx); - return tpl; -} - -#define APPEND_OLD_CHUNK \ - if (chunk != NULL) { \ - if (chunks == NULL) { \ - chunks = PyList_New(0); \ - if (chunks == NULL) { \ - goto bail; \ - } \ - } \ - if (PyList_Append(chunks, chunk)) { \ - goto bail; \ - } \ - Py_CLEAR(chunk); \ - } - -#if PY_MAJOR_VERSION < 3 -static PyObject * -scanstring_str(PyObject *pystr, Py_ssize_t end, char *encoding, int strict, Py_ssize_t *next_end_ptr) -{ - /* Read the JSON string from PyString pystr. - end is the index of the first character after the quote. - encoding is the encoding of pystr (must be an ASCII superset) - if strict is zero then literal control characters are allowed - *next_end_ptr is a return-by-reference index of the character - after the end quote - - Return value is a new PyString (if ASCII-only) or PyUnicode - */ - PyObject *rval; - Py_ssize_t len = PyString_GET_SIZE(pystr); - Py_ssize_t begin = end - 1; - Py_ssize_t next = begin; - int has_unicode = 0; - char *buf = PyString_AS_STRING(pystr); - PyObject *chunks = NULL; - PyObject *chunk = NULL; - PyObject *strchunk = NULL; - - if (len == end) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - else if (end < 0 || len < end) { - PyErr_SetString(PyExc_ValueError, "end is out of bounds"); - goto bail; - } - while (1) { - /* Find the end of the string or the next escape */ - Py_UNICODE c = 0; - for (next = end; next < len; next++) { - c = (unsigned char)buf[next]; - if (c == '"' || c == '\\') { - break; - } - else if (strict && c <= 0x1f) { - raise_errmsg(ERR_STRING_CONTROL, pystr, next); - goto bail; - } - else if (c > 0x7f) { - has_unicode = 1; - } - } - if (!(c == '"' || c == '\\')) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - /* Pick up this chunk if it's not zero length */ - if (next != end) { - APPEND_OLD_CHUNK - strchunk = PyString_FromStringAndSize(&buf[end], next - end); - if (strchunk == NULL) { - goto bail; - } - if (has_unicode) { - chunk = PyUnicode_FromEncodedObject(strchunk, encoding, NULL); - Py_DECREF(strchunk); - if (chunk == NULL) { - goto bail; - } - } - else { - chunk = strchunk; - } - } - next++; - if (c == '"') { - end = next; - break; - } - if (next == len) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - c = buf[next]; - if (c != 'u') { - /* Non-unicode backslash escapes */ - end = next + 1; - switch (c) { - case '"': break; - case '\\': break; - case '/': break; - case 'b': c = '\b'; break; - case 'f': c = '\f'; break; - case 'n': c = '\n'; break; - case 'r': c = '\r'; break; - case 't': c = '\t'; break; - default: c = 0; - } - if (c == 0) { - raise_errmsg(ERR_STRING_ESC1, pystr, end - 2); - goto bail; - } - } - else { - c = 0; - next++; - end = next + 4; - if (end >= len) { - raise_errmsg(ERR_STRING_ESC4, pystr, next - 1); - goto bail; - } - /* Decode 4 hex digits */ - for (; next < end; next++) { - JSON_UNICHR digit = (JSON_UNICHR)buf[next]; - c <<= 4; - switch (digit) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - c |= (digit - '0'); break; - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': - c |= (digit - 'a' + 10); break; - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': - c |= (digit - 'A' + 10); break; - default: - raise_errmsg(ERR_STRING_ESC4, pystr, end - 5); - goto bail; - } - } -#if defined(Py_UNICODE_WIDE) - /* Surrogate pair */ - if ((c & 0xfc00) == 0xd800) { - if (end + 6 < len && buf[next] == '\\' && buf[next+1] == 'u') { - JSON_UNICHR c2 = 0; - end += 6; - /* Decode 4 hex digits */ - for (next += 2; next < end; next++) { - c2 <<= 4; - JSON_UNICHR digit = buf[next]; - switch (digit) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - c2 |= (digit - '0'); break; - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': - c2 |= (digit - 'a' + 10); break; - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': - c2 |= (digit - 'A' + 10); break; - default: - raise_errmsg(ERR_STRING_ESC4, pystr, end - 5); - goto bail; - } - } - if ((c2 & 0xfc00) != 0xdc00) { - /* not a low surrogate, rewind */ - end -= 6; - next = end; - } - else { - c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00)); - } - } - } -#endif /* Py_UNICODE_WIDE */ - } - if (c > 0x7f) { - has_unicode = 1; - } - APPEND_OLD_CHUNK - if (has_unicode) { - chunk = PyUnicode_FromOrdinal(c); - if (chunk == NULL) { - goto bail; - } - } - else { - char c_char = Py_CHARMASK(c); - chunk = PyString_FromStringAndSize(&c_char, 1); - if (chunk == NULL) { - goto bail; - } - } - } - - if (chunks == NULL) { - if (chunk != NULL) - rval = chunk; - else { - rval = JSON_EmptyStr; - Py_INCREF(rval); - } - } - else { - APPEND_OLD_CHUNK - rval = join_list_string(chunks); - if (rval == NULL) { - goto bail; - } - Py_CLEAR(chunks); - } - - *next_end_ptr = end; - return rval; -bail: - *next_end_ptr = -1; - Py_XDECREF(chunk); - Py_XDECREF(chunks); - return NULL; -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next_end_ptr) -{ - /* Read the JSON string from PyUnicode pystr. - end is the index of the first character after the quote. - if strict is zero then literal control characters are allowed - *next_end_ptr is a return-by-reference index of the character - after the end quote - - Return value is a new PyUnicode - */ - PyObject *rval; - Py_ssize_t begin = end - 1; - Py_ssize_t next = begin; - PY2_UNUSED int kind = PyUnicode_KIND(pystr); - Py_ssize_t len = PyUnicode_GET_LENGTH(pystr); - void *buf = PyUnicode_DATA(pystr); - PyObject *chunks = NULL; - PyObject *chunk = NULL; - - if (len == end) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - else if (end < 0 || len < end) { - PyErr_SetString(PyExc_ValueError, "end is out of bounds"); - goto bail; - } - while (1) { - /* Find the end of the string or the next escape */ - JSON_UNICHR c = 0; - for (next = end; next < len; next++) { - c = PyUnicode_READ(kind, buf, next); - if (c == '"' || c == '\\') { - break; - } - else if (strict && c <= 0x1f) { - raise_errmsg(ERR_STRING_CONTROL, pystr, next); - goto bail; - } - } - if (!(c == '"' || c == '\\')) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - /* Pick up this chunk if it's not zero length */ - if (next != end) { - APPEND_OLD_CHUNK -#if PY_MAJOR_VERSION < 3 - chunk = PyUnicode_FromUnicode(&((const Py_UNICODE *)buf)[end], next - end); -#else - chunk = PyUnicode_Substring(pystr, end, next); -#endif - if (chunk == NULL) { - goto bail; - } - } - next++; - if (c == '"') { - end = next; - break; - } - if (next == len) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - c = PyUnicode_READ(kind, buf, next); - if (c != 'u') { - /* Non-unicode backslash escapes */ - end = next + 1; - switch (c) { - case '"': break; - case '\\': break; - case '/': break; - case 'b': c = '\b'; break; - case 'f': c = '\f'; break; - case 'n': c = '\n'; break; - case 'r': c = '\r'; break; - case 't': c = '\t'; break; - default: c = 0; - } - if (c == 0) { - raise_errmsg(ERR_STRING_ESC1, pystr, end - 2); - goto bail; - } - } - else { - c = 0; - next++; - end = next + 4; - if (end >= len) { - raise_errmsg(ERR_STRING_ESC4, pystr, next - 1); - goto bail; - } - /* Decode 4 hex digits */ - for (; next < end; next++) { - JSON_UNICHR digit = PyUnicode_READ(kind, buf, next); - c <<= 4; - switch (digit) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - c |= (digit - '0'); break; - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': - c |= (digit - 'a' + 10); break; - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': - c |= (digit - 'A' + 10); break; - default: - raise_errmsg(ERR_STRING_ESC4, pystr, end - 5); - goto bail; - } - } -#if PY_MAJOR_VERSION >= 3 || defined(Py_UNICODE_WIDE) - /* Surrogate pair */ - if ((c & 0xfc00) == 0xd800) { - JSON_UNICHR c2 = 0; - if (end + 6 < len && - PyUnicode_READ(kind, buf, next) == '\\' && - PyUnicode_READ(kind, buf, next + 1) == 'u') { - end += 6; - /* Decode 4 hex digits */ - for (next += 2; next < end; next++) { - JSON_UNICHR digit = PyUnicode_READ(kind, buf, next); - c2 <<= 4; - switch (digit) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - c2 |= (digit - '0'); break; - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': - c2 |= (digit - 'a' + 10); break; - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': - c2 |= (digit - 'A' + 10); break; - default: - raise_errmsg(ERR_STRING_ESC4, pystr, end - 5); - goto bail; - } - } - if ((c2 & 0xfc00) != 0xdc00) { - /* not a low surrogate, rewind */ - end -= 6; - next = end; - } - else { - c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00)); - } - } - } -#endif - } - APPEND_OLD_CHUNK - chunk = PyUnicode_FromOrdinal(c); - if (chunk == NULL) { - goto bail; - } - } - - if (chunks == NULL) { - if (chunk != NULL) - rval = chunk; - else { - rval = JSON_EmptyUnicode; - Py_INCREF(rval); - } - } - else { - APPEND_OLD_CHUNK - rval = join_list_unicode(chunks); - if (rval == NULL) { - goto bail; - } - Py_CLEAR(chunks); - } - *next_end_ptr = end; - return rval; -bail: - *next_end_ptr = -1; - Py_XDECREF(chunk); - Py_XDECREF(chunks); - return NULL; -} - -PyDoc_STRVAR(pydoc_scanstring, - "scanstring(basestring, end, encoding, strict=True) -> (str, end)\n" - "\n" - "Scan the string s for a JSON string. End is the index of the\n" - "character in s after the quote that started the JSON string.\n" - "Unescapes all valid JSON string escape sequences and raises ValueError\n" - "on attempt to decode an invalid string. If strict is False then literal\n" - "control characters are allowed in the string.\n" - "\n" - "Returns a tuple of the decoded string and the index of the character in s\n" - "after the end quote." -); - -static PyObject * -py_scanstring(PyObject* self UNUSED, PyObject *args) -{ - PyObject *pystr; - PyObject *rval; - Py_ssize_t end; - Py_ssize_t next_end = -1; - char *encoding = NULL; - int strict = 1; - if (!PyArg_ParseTuple(args, "OO&|zi:scanstring", &pystr, _convertPyInt_AsSsize_t, &end, &encoding, &strict)) { - return NULL; - } - if (encoding == NULL) { - encoding = DEFAULT_ENCODING; - } - if (PyUnicode_Check(pystr)) { - if (PyUnicode_READY(pystr)) - return NULL; - rval = scanstring_unicode(pystr, end, strict, &next_end); - } -#if PY_MAJOR_VERSION < 3 - /* Using a bytes input is unsupported for scanning in Python 3. - It is coerced to str in the decoder before it gets here. */ - else if (PyString_Check(pystr)) { - rval = scanstring_str(pystr, end, encoding, strict, &next_end); - } -#endif - else { - PyErr_Format(PyExc_TypeError, - "first argument must be a string, not %.80s", - Py_TYPE(pystr)->tp_name); - return NULL; - } - return _build_rval_index_tuple(rval, next_end); -} - -PyDoc_STRVAR(pydoc_encode_basestring_ascii, - "encode_basestring_ascii(basestring) -> str\n" - "\n" - "Return an ASCII-only JSON representation of a Python string" -); - -static PyObject * -py_encode_basestring_ascii(PyObject* self UNUSED, PyObject *pystr) -{ - /* Return an ASCII-only JSON representation of a Python string */ - /* METH_O */ - if (PyBytes_Check(pystr)) { - return ascii_escape_str(pystr); - } - else if (PyUnicode_Check(pystr)) { - if (PyUnicode_READY(pystr)) - return NULL; - return ascii_escape_unicode(pystr); - } - else { - PyErr_Format(PyExc_TypeError, - "first argument must be a string, not %.80s", - Py_TYPE(pystr)->tp_name); - return NULL; - } -} - -static void -scanner_dealloc(PyObject *self) -{ - /* bpo-31095: UnTrack is needed before calling any callbacks */ - PyObject_GC_UnTrack(self); - scanner_clear(self); - Py_TYPE(self)->tp_free(self); -} - -static int -scanner_traverse(PyObject *self, visitproc visit, void *arg) -{ - PyScannerObject *s; - assert(PyScanner_Check(self)); - s = (PyScannerObject *)self; - Py_VISIT(s->encoding); - Py_VISIT(s->strict_bool); - Py_VISIT(s->object_hook); - Py_VISIT(s->pairs_hook); - Py_VISIT(s->parse_float); - Py_VISIT(s->parse_int); - Py_VISIT(s->parse_constant); - Py_VISIT(s->memo); - return 0; -} - -static int -scanner_clear(PyObject *self) -{ - PyScannerObject *s; - assert(PyScanner_Check(self)); - s = (PyScannerObject *)self; - Py_CLEAR(s->encoding); - Py_CLEAR(s->strict_bool); - Py_CLEAR(s->object_hook); - Py_CLEAR(s->pairs_hook); - Py_CLEAR(s->parse_float); - Py_CLEAR(s->parse_int); - Py_CLEAR(s->parse_constant); - Py_CLEAR(s->memo); - return 0; -} - -#if PY_MAJOR_VERSION < 3 -static PyObject * -_parse_object_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON object from PyString pystr. - idx is the index of the first character after the opening curly brace. - *next_idx_ptr is a return-by-reference index to the first character after - the closing curly brace. - - Returns a new PyObject (usually a dict, but object_hook or - object_pairs_hook can change that) - */ - char *str = PyString_AS_STRING(pystr); - Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1; - PyObject *rval = NULL; - PyObject *pairs = NULL; - PyObject *item; - PyObject *key = NULL; - PyObject *val = NULL; - char *encoding = PyString_AS_STRING(s->encoding); - int has_pairs_hook = (s->pairs_hook != Py_None); - int did_parse = 0; - Py_ssize_t next_idx; - if (has_pairs_hook) { - pairs = PyList_New(0); - if (pairs == NULL) - return NULL; - } - else { - rval = PyDict_New(); - if (rval == NULL) - return NULL; - } - - /* skip whitespace after { */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* only loop if the object is non-empty */ - if (idx <= end_idx && str[idx] != '}') { - int trailing_delimiter = 0; - while (idx <= end_idx) { - PyObject *memokey; - trailing_delimiter = 0; - - /* read key */ - if (str[idx] != '"') { - raise_errmsg(ERR_OBJECT_PROPERTY, pystr, idx); - goto bail; - } - key = scanstring_str(pystr, idx + 1, encoding, s->strict, &next_idx); - if (key == NULL) - goto bail; - memokey = PyDict_GetItem(s->memo, key); - if (memokey != NULL) { - Py_INCREF(memokey); - Py_DECREF(key); - key = memokey; - } - else { - if (PyDict_SetItem(s->memo, key, key) < 0) - goto bail; - } - idx = next_idx; - - /* skip whitespace between key and : delimiter, read :, skip whitespace */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - if (idx > end_idx || str[idx] != ':') { - raise_errmsg(ERR_OBJECT_PROPERTY_DELIMITER, pystr, idx); - goto bail; - } - idx++; - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* read any JSON data type */ - val = scan_once_str(s, pystr, idx, &next_idx); - if (val == NULL) - goto bail; - - if (has_pairs_hook) { - item = PyTuple_Pack(2, key, val); - if (item == NULL) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - if (PyList_Append(pairs, item) == -1) { - Py_DECREF(item); - goto bail; - } - Py_DECREF(item); - } - else { - if (PyDict_SetItem(rval, key, val) < 0) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - } - idx = next_idx; - - /* skip whitespace before } or , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* bail if the object is closed or we didn't get the , delimiter */ - did_parse = 1; - if (idx > end_idx) break; - if (str[idx] == '}') { - break; - } - else if (str[idx] != ',') { - raise_errmsg(ERR_OBJECT_DELIMITER, pystr, idx); - goto bail; - } - idx++; - - /* skip whitespace after , delimiter */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - trailing_delimiter = 1; - } - if (trailing_delimiter) { - raise_errmsg(ERR_OBJECT_PROPERTY, pystr, idx); - goto bail; - } - } - /* verify that idx < end_idx, str[idx] should be '}' */ - if (idx > end_idx || str[idx] != '}') { - if (did_parse) { - raise_errmsg(ERR_OBJECT_DELIMITER, pystr, idx); - } else { - raise_errmsg(ERR_OBJECT_PROPERTY_FIRST, pystr, idx); - } - goto bail; - } - - /* if pairs_hook is not None: rval = object_pairs_hook(pairs) */ - if (s->pairs_hook != Py_None) { - val = PyObject_CallOneArg(s->pairs_hook, pairs); - if (val == NULL) - goto bail; - Py_DECREF(pairs); - *next_idx_ptr = idx + 1; - return val; - } - - /* if object_hook is not None: rval = object_hook(rval) */ - if (s->object_hook != Py_None) { - val = PyObject_CallOneArg(s->object_hook, rval); - if (val == NULL) - goto bail; - Py_DECREF(rval); - rval = val; - val = NULL; - } - *next_idx_ptr = idx + 1; - return rval; -bail: - Py_XDECREF(rval); - Py_XDECREF(key); - Py_XDECREF(val); - Py_XDECREF(pairs); - return NULL; -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -_parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON object from PyUnicode pystr. - idx is the index of the first character after the opening curly brace. - *next_idx_ptr is a return-by-reference index to the first character after - the closing curly brace. - - Returns a new PyObject (usually a dict, but object_hook can change that) - */ - void *str = PyUnicode_DATA(pystr); - Py_ssize_t end_idx = PyUnicode_GET_LENGTH(pystr) - 1; - PY2_UNUSED int kind = PyUnicode_KIND(pystr); - PyObject *rval = NULL; - PyObject *pairs = NULL; - PyObject *item; - PyObject *key = NULL; - PyObject *val = NULL; - int has_pairs_hook = (s->pairs_hook != Py_None); - int did_parse = 0; - Py_ssize_t next_idx; - - if (has_pairs_hook) { - pairs = PyList_New(0); - if (pairs == NULL) - return NULL; - } - else { - rval = PyDict_New(); - if (rval == NULL) - return NULL; - } - - /* skip whitespace after { */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - - /* only loop if the object is non-empty */ - if (idx <= end_idx && PyUnicode_READ(kind, str, idx) != '}') { - int trailing_delimiter = 0; - while (idx <= end_idx) { - PyObject *memokey; - trailing_delimiter = 0; - - /* read key */ - if (PyUnicode_READ(kind, str, idx) != '"') { - raise_errmsg(ERR_OBJECT_PROPERTY, pystr, idx); - goto bail; - } - key = scanstring_unicode(pystr, idx + 1, s->strict, &next_idx); - if (key == NULL) - goto bail; - memokey = PyDict_GetItem(s->memo, key); - if (memokey != NULL) { - Py_INCREF(memokey); - Py_DECREF(key); - key = memokey; - } - else { - if (PyDict_SetItem(s->memo, key, key) < 0) - goto bail; - } - idx = next_idx; - - /* skip whitespace between key and : delimiter, read :, skip - whitespace */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - if (idx > end_idx || PyUnicode_READ(kind, str, idx) != ':') { - raise_errmsg(ERR_OBJECT_PROPERTY_DELIMITER, pystr, idx); - goto bail; - } - idx++; - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - - /* read any JSON term */ - val = scan_once_unicode(s, pystr, idx, &next_idx); - if (val == NULL) - goto bail; - - if (has_pairs_hook) { - item = PyTuple_Pack(2, key, val); - if (item == NULL) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - if (PyList_Append(pairs, item) == -1) { - Py_DECREF(item); - goto bail; - } - Py_DECREF(item); - } - else { - if (PyDict_SetItem(rval, key, val) < 0) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - } - idx = next_idx; - - /* skip whitespace before } or , */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - - /* bail if the object is closed or we didn't get the , - delimiter */ - did_parse = 1; - if (idx > end_idx) break; - if (PyUnicode_READ(kind, str, idx) == '}') { - break; - } - else if (PyUnicode_READ(kind, str, idx) != ',') { - raise_errmsg(ERR_OBJECT_DELIMITER, pystr, idx); - goto bail; - } - idx++; - - /* skip whitespace after , delimiter */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - trailing_delimiter = 1; - } - if (trailing_delimiter) { - raise_errmsg(ERR_OBJECT_PROPERTY, pystr, idx); - goto bail; - } - } - - /* verify that idx < end_idx, str[idx] should be '}' */ - if (idx > end_idx || PyUnicode_READ(kind, str, idx) != '}') { - if (did_parse) { - raise_errmsg(ERR_OBJECT_DELIMITER, pystr, idx); - } else { - raise_errmsg(ERR_OBJECT_PROPERTY_FIRST, pystr, idx); - } - goto bail; - } - - /* if pairs_hook is not None: rval = object_pairs_hook(pairs) */ - if (s->pairs_hook != Py_None) { - val = PyObject_CallOneArg(s->pairs_hook, pairs); - if (val == NULL) - goto bail; - Py_DECREF(pairs); - *next_idx_ptr = idx + 1; - return val; - } - - /* if object_hook is not None: rval = object_hook(rval) */ - if (s->object_hook != Py_None) { - val = PyObject_CallOneArg(s->object_hook, rval); - if (val == NULL) - goto bail; - Py_DECREF(rval); - rval = val; - val = NULL; - } - *next_idx_ptr = idx + 1; - return rval; -bail: - Py_XDECREF(rval); - Py_XDECREF(key); - Py_XDECREF(val); - Py_XDECREF(pairs); - return NULL; -} - -#if PY_MAJOR_VERSION < 3 -static PyObject * -_parse_array_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON array from PyString pystr. - idx is the index of the first character after the opening brace. - *next_idx_ptr is a return-by-reference index to the first character after - the closing brace. - - Returns a new PyList - */ - char *str = PyString_AS_STRING(pystr); - Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1; - PyObject *val = NULL; - PyObject *rval = PyList_New(0); - Py_ssize_t next_idx; - if (rval == NULL) - return NULL; - - /* skip whitespace after [ */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* only loop if the array is non-empty */ - if (idx <= end_idx && str[idx] != ']') { - int trailing_delimiter = 0; - while (idx <= end_idx) { - trailing_delimiter = 0; - /* read any JSON term and de-tuplefy the (rval, idx) */ - val = scan_once_str(s, pystr, idx, &next_idx); - if (val == NULL) { - goto bail; - } - - if (PyList_Append(rval, val) == -1) - goto bail; - - Py_CLEAR(val); - idx = next_idx; - - /* skip whitespace between term and , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* bail if the array is closed or we didn't get the , delimiter */ - if (idx > end_idx) break; - if (str[idx] == ']') { - break; - } - else if (str[idx] != ',') { - raise_errmsg(ERR_ARRAY_DELIMITER, pystr, idx); - goto bail; - } - idx++; - - /* skip whitespace after , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - trailing_delimiter = 1; - } - if (trailing_delimiter) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx); - goto bail; - } - } - - /* verify that idx < end_idx, str[idx] should be ']' */ - if (idx > end_idx || str[idx] != ']') { - if (PyList_GET_SIZE(rval)) { - raise_errmsg(ERR_ARRAY_DELIMITER, pystr, idx); - } else { - raise_errmsg(ERR_ARRAY_VALUE_FIRST, pystr, idx); - } - goto bail; - } - *next_idx_ptr = idx + 1; - return rval; -bail: - Py_XDECREF(val); - Py_DECREF(rval); - return NULL; -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -_parse_array_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON array from PyString pystr. - idx is the index of the first character after the opening brace. - *next_idx_ptr is a return-by-reference index to the first character after - the closing brace. - - Returns a new PyList - */ - PY2_UNUSED int kind = PyUnicode_KIND(pystr); - void *str = PyUnicode_DATA(pystr); - Py_ssize_t end_idx = PyUnicode_GET_LENGTH(pystr) - 1; - PyObject *val = NULL; - PyObject *rval = PyList_New(0); - Py_ssize_t next_idx; - if (rval == NULL) - return NULL; - - /* skip whitespace after [ */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - - /* only loop if the array is non-empty */ - if (idx <= end_idx && PyUnicode_READ(kind, str, idx) != ']') { - int trailing_delimiter = 0; - while (idx <= end_idx) { - trailing_delimiter = 0; - /* read any JSON term */ - val = scan_once_unicode(s, pystr, idx, &next_idx); - if (val == NULL) { - goto bail; - } - - if (PyList_Append(rval, val) == -1) - goto bail; - - Py_CLEAR(val); - idx = next_idx; - - /* skip whitespace between term and , */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - - /* bail if the array is closed or we didn't get the , delimiter */ - if (idx > end_idx) break; - if (PyUnicode_READ(kind, str, idx) == ']') { - break; - } - else if (PyUnicode_READ(kind, str, idx) != ',') { - raise_errmsg(ERR_ARRAY_DELIMITER, pystr, idx); - goto bail; - } - idx++; - - /* skip whitespace after , */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - trailing_delimiter = 1; - } - if (trailing_delimiter) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx); - goto bail; - } - } - - /* verify that idx < end_idx, str[idx] should be ']' */ - if (idx > end_idx || PyUnicode_READ(kind, str, idx) != ']') { - if (PyList_GET_SIZE(rval)) { - raise_errmsg(ERR_ARRAY_DELIMITER, pystr, idx); - } else { - raise_errmsg(ERR_ARRAY_VALUE_FIRST, pystr, idx); - } - goto bail; - } - *next_idx_ptr = idx + 1; - return rval; -bail: - Py_XDECREF(val); - Py_DECREF(rval); - return NULL; -} - -static PyObject * -_parse_constant(PyScannerObject *s, PyObject *pystr, PyObject *constant, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON constant from PyString pystr. - constant is the Python string that was found - ("NaN", "Infinity", "-Infinity"). - idx is the index of the first character of the constant - *next_idx_ptr is a return-by-reference index to the first character after - the constant. - - Returns the result of parse_constant - */ - PyObject *rval; - if (s->parse_constant == Py_None) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx); - return NULL; - } - - /* rval = parse_constant(constant) */ - rval = PyObject_CallOneArg(s->parse_constant, constant); - idx += PyString_GET_SIZE(constant); - *next_idx_ptr = idx; - return rval; -} - -#if PY_MAJOR_VERSION < 3 -static PyObject * -_match_number_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON number from PyString pystr. - idx is the index of the first character of the number - *next_idx_ptr is a return-by-reference index to the first character after - the number. - - Returns a new PyObject representation of that number: - PyInt, PyLong, or PyFloat. - May return other types if parse_int or parse_float are set - */ - char *str = PyString_AS_STRING(pystr); - Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1; - Py_ssize_t idx = start; - int is_float = 0; - PyObject *rval; - PyObject *numstr; - - /* read a sign if it's there, make sure it's not the end of the string */ - if (str[idx] == '-') { - if (idx >= end_idx) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, start); - return NULL; - } - idx++; - } - - /* read as many integer digits as we find as long as it doesn't start with 0 */ - if (str[idx] >= '1' && str[idx] <= '9') { - idx++; - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; - } - /* if it starts with 0 we only expect one integer digit */ - else if (str[idx] == '0') { - idx++; - } - /* no integer digits, error */ - else { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, start); - return NULL; - } - - /* if the next char is '.' followed by a digit then read all float digits */ - if (idx < end_idx && str[idx] == '.' && str[idx + 1] >= '0' && str[idx + 1] <= '9') { - is_float = 1; - idx += 2; - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; - } - - /* if the next char is 'e' or 'E' then maybe read the exponent (or backtrack) */ - if (idx < end_idx && (str[idx] == 'e' || str[idx] == 'E')) { - - /* save the index of the 'e' or 'E' just in case we need to backtrack */ - Py_ssize_t e_start = idx; - idx++; - - /* read an exponent sign if present */ - if (idx < end_idx && (str[idx] == '-' || str[idx] == '+')) idx++; - - /* read all digits */ - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; - - /* if we got a digit, then parse as float. if not, backtrack */ - if (str[idx - 1] >= '0' && str[idx - 1] <= '9') { - is_float = 1; - } - else { - idx = e_start; - } - } - - /* copy the section we determined to be a number */ - numstr = PyString_FromStringAndSize(&str[start], idx - start); - if (numstr == NULL) - return NULL; - if (is_float) { - /* parse as a float using a fast path if available, otherwise call user defined method */ - if (s->parse_float != (PyObject *)&PyFloat_Type) { - rval = PyObject_CallOneArg(s->parse_float, numstr); - } - else { - /* rval = PyFloat_FromDouble(PyOS_ascii_atof(PyString_AS_STRING(numstr))); */ - double d = PyOS_string_to_double(PyString_AS_STRING(numstr), - NULL, NULL); - if (d == -1.0 && PyErr_Occurred()) { - Py_DECREF(numstr); - return NULL; - } - rval = PyFloat_FromDouble(d); - } - } - else { - /* parse as an int using a fast path if available, otherwise call user defined method */ - if (s->parse_int != (PyObject *)&PyInt_Type) { - rval = PyObject_CallOneArg(s->parse_int, numstr); - } - else { - rval = PyInt_FromString(PyString_AS_STRING(numstr), NULL, 10); - } - } - Py_DECREF(numstr); - *next_idx_ptr = idx; - return rval; -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -_match_number_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON number from PyUnicode pystr. - idx is the index of the first character of the number - *next_idx_ptr is a return-by-reference index to the first character after - the number. - - Returns a new PyObject representation of that number: - PyInt, PyLong, or PyFloat. - May return other types if parse_int or parse_float are set - */ - PY2_UNUSED int kind = PyUnicode_KIND(pystr); - void *str = PyUnicode_DATA(pystr); - Py_ssize_t end_idx = PyUnicode_GET_LENGTH(pystr) - 1; - Py_ssize_t idx = start; - int is_float = 0; - JSON_UNICHR c; - PyObject *rval; - PyObject *numstr; - - /* read a sign if it's there, make sure it's not the end of the string */ - if (PyUnicode_READ(kind, str, idx) == '-') { - if (idx >= end_idx) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, start); - return NULL; - } - idx++; - } - - /* read as many integer digits as we find as long as it doesn't start with 0 */ - c = PyUnicode_READ(kind, str, idx); - if (c == '0') { - /* if it starts with 0 we only expect one integer digit */ - idx++; - } - else if (IS_DIGIT(c)) { - idx++; - while (idx <= end_idx && IS_DIGIT(PyUnicode_READ(kind, str, idx))) { - idx++; - } - } - else { - /* no integer digits, error */ - raise_errmsg(ERR_EXPECTING_VALUE, pystr, start); - return NULL; - } - - /* if the next char is '.' followed by a digit then read all float digits */ - if (idx < end_idx && - PyUnicode_READ(kind, str, idx) == '.' && - IS_DIGIT(PyUnicode_READ(kind, str, idx + 1))) { - is_float = 1; - idx += 2; - while (idx <= end_idx && IS_DIGIT(PyUnicode_READ(kind, str, idx))) idx++; - } - - /* if the next char is 'e' or 'E' then maybe read the exponent (or backtrack) */ - if (idx < end_idx && - (PyUnicode_READ(kind, str, idx) == 'e' || - PyUnicode_READ(kind, str, idx) == 'E')) { - Py_ssize_t e_start = idx; - idx++; - - /* read an exponent sign if present */ - if (idx < end_idx && - (PyUnicode_READ(kind, str, idx) == '-' || - PyUnicode_READ(kind, str, idx) == '+')) idx++; - - /* read all digits */ - while (idx <= end_idx && IS_DIGIT(PyUnicode_READ(kind, str, idx))) idx++; - - /* if we got a digit, then parse as float. if not, backtrack */ - if (IS_DIGIT(PyUnicode_READ(kind, str, idx - 1))) { - is_float = 1; - } - else { - idx = e_start; - } - } - - /* copy the section we determined to be a number */ -#if PY_MAJOR_VERSION >= 3 - numstr = PyUnicode_Substring(pystr, start, idx); -#else - numstr = PyUnicode_FromUnicode(&((Py_UNICODE *)str)[start], idx - start); -#endif - if (numstr == NULL) - return NULL; - if (is_float) { - /* parse as a float using a fast path if available, otherwise call user defined method */ - if (s->parse_float != (PyObject *)&PyFloat_Type) { - rval = PyObject_CallOneArg(s->parse_float, numstr); - } - else { -#if PY_MAJOR_VERSION >= 3 - rval = PyFloat_FromString(numstr); -#else - rval = PyFloat_FromString(numstr, NULL); -#endif - } - } - else { - /* no fast path for unicode -> int, just call */ - rval = PyObject_CallOneArg(s->parse_int, numstr); - } - Py_DECREF(numstr); - *next_idx_ptr = idx; - return rval; -} - -#if PY_MAJOR_VERSION < 3 -static PyObject * -scan_once_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read one JSON term (of any kind) from PyString pystr. - idx is the index of the first character of the term - *next_idx_ptr is a return-by-reference index to the first character after - the number. - - Returns a new PyObject representation of the term. - */ - char *str = PyString_AS_STRING(pystr); - Py_ssize_t length = PyString_GET_SIZE(pystr); - PyObject *rval = NULL; - int fallthrough = 0; - if (idx < 0 || idx >= length) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx); - return NULL; - } - switch (str[idx]) { - case '"': - /* string */ - rval = scanstring_str(pystr, idx + 1, - PyString_AS_STRING(s->encoding), - s->strict, - next_idx_ptr); - break; - case '{': - /* object */ - if (Py_EnterRecursiveCall(" while decoding a JSON object " - "from a string")) - return NULL; - rval = _parse_object_str(s, pystr, idx + 1, next_idx_ptr); - Py_LeaveRecursiveCall(); - break; - case '[': - /* array */ - if (Py_EnterRecursiveCall(" while decoding a JSON array " - "from a string")) - return NULL; - rval = _parse_array_str(s, pystr, idx + 1, next_idx_ptr); - Py_LeaveRecursiveCall(); - break; - case 'n': - /* null */ - if ((idx + 3 < length) && str[idx + 1] == 'u' && str[idx + 2] == 'l' && str[idx + 3] == 'l') { - Py_INCREF(Py_None); - *next_idx_ptr = idx + 4; - rval = Py_None; - } - else - fallthrough = 1; - break; - case 't': - /* true */ - if ((idx + 3 < length) && str[idx + 1] == 'r' && str[idx + 2] == 'u' && str[idx + 3] == 'e') { - Py_INCREF(Py_True); - *next_idx_ptr = idx + 4; - rval = Py_True; - } - else - fallthrough = 1; - break; - case 'f': - /* false */ - if ((idx + 4 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'l' && str[idx + 3] == 's' && str[idx + 4] == 'e') { - Py_INCREF(Py_False); - *next_idx_ptr = idx + 5; - rval = Py_False; - } - else - fallthrough = 1; - break; - case 'N': - /* NaN */ - if ((idx + 2 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'N') { - rval = _parse_constant(s, pystr, JSON_NaN, idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - case 'I': - /* Infinity */ - if ((idx + 7 < length) && str[idx + 1] == 'n' && str[idx + 2] == 'f' && str[idx + 3] == 'i' && str[idx + 4] == 'n' && str[idx + 5] == 'i' && str[idx + 6] == 't' && str[idx + 7] == 'y') { - rval = _parse_constant(s, pystr, JSON_Infinity, idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - case '-': - /* -Infinity */ - if ((idx + 8 < length) && str[idx + 1] == 'I' && str[idx + 2] == 'n' && str[idx + 3] == 'f' && str[idx + 4] == 'i' && str[idx + 5] == 'n' && str[idx + 6] == 'i' && str[idx + 7] == 't' && str[idx + 8] == 'y') { - rval = _parse_constant(s, pystr, JSON_NegInfinity, idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - default: - fallthrough = 1; - } - /* Didn't find a string, object, array, or named constant. Look for a number. */ - if (fallthrough) - rval = _match_number_str(s, pystr, idx, next_idx_ptr); - return rval; -} -#endif /* PY_MAJOR_VERSION < 3 */ - - -static PyObject * -scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read one JSON term (of any kind) from PyUnicode pystr. - idx is the index of the first character of the term - *next_idx_ptr is a return-by-reference index to the first character after - the number. - - Returns a new PyObject representation of the term. - */ - PY2_UNUSED int kind = PyUnicode_KIND(pystr); - void *str = PyUnicode_DATA(pystr); - Py_ssize_t length = PyUnicode_GET_LENGTH(pystr); - PyObject *rval = NULL; - int fallthrough = 0; - if (idx < 0 || idx >= length) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx); - return NULL; - } - switch (PyUnicode_READ(kind, str, idx)) { - case '"': - /* string */ - rval = scanstring_unicode(pystr, idx + 1, - s->strict, - next_idx_ptr); - break; - case '{': - /* object */ - if (Py_EnterRecursiveCall(" while decoding a JSON object " - "from a unicode string")) - return NULL; - rval = _parse_object_unicode(s, pystr, idx + 1, next_idx_ptr); - Py_LeaveRecursiveCall(); - break; - case '[': - /* array */ - if (Py_EnterRecursiveCall(" while decoding a JSON array " - "from a unicode string")) - return NULL; - rval = _parse_array_unicode(s, pystr, idx + 1, next_idx_ptr); - Py_LeaveRecursiveCall(); - break; - case 'n': - /* null */ - if ((idx + 3 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'u' && - PyUnicode_READ(kind, str, idx + 2) == 'l' && - PyUnicode_READ(kind, str, idx + 3) == 'l') { - Py_INCREF(Py_None); - *next_idx_ptr = idx + 4; - rval = Py_None; - } - else - fallthrough = 1; - break; - case 't': - /* true */ - if ((idx + 3 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'r' && - PyUnicode_READ(kind, str, idx + 2) == 'u' && - PyUnicode_READ(kind, str, idx + 3) == 'e') { - Py_INCREF(Py_True); - *next_idx_ptr = idx + 4; - rval = Py_True; - } - else - fallthrough = 1; - break; - case 'f': - /* false */ - if ((idx + 4 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'a' && - PyUnicode_READ(kind, str, idx + 2) == 'l' && - PyUnicode_READ(kind, str, idx + 3) == 's' && - PyUnicode_READ(kind, str, idx + 4) == 'e') { - Py_INCREF(Py_False); - *next_idx_ptr = idx + 5; - rval = Py_False; - } - else - fallthrough = 1; - break; - case 'N': - /* NaN */ - if ((idx + 2 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'a' && - PyUnicode_READ(kind, str, idx + 2) == 'N') { - rval = _parse_constant(s, pystr, JSON_NaN, idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - case 'I': - /* Infinity */ - if ((idx + 7 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'n' && - PyUnicode_READ(kind, str, idx + 2) == 'f' && - PyUnicode_READ(kind, str, idx + 3) == 'i' && - PyUnicode_READ(kind, str, idx + 4) == 'n' && - PyUnicode_READ(kind, str, idx + 5) == 'i' && - PyUnicode_READ(kind, str, idx + 6) == 't' && - PyUnicode_READ(kind, str, idx + 7) == 'y') { - rval = _parse_constant(s, pystr, JSON_Infinity, idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - case '-': - /* -Infinity */ - if ((idx + 8 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'I' && - PyUnicode_READ(kind, str, idx + 2) == 'n' && - PyUnicode_READ(kind, str, idx + 3) == 'f' && - PyUnicode_READ(kind, str, idx + 4) == 'i' && - PyUnicode_READ(kind, str, idx + 5) == 'n' && - PyUnicode_READ(kind, str, idx + 6) == 'i' && - PyUnicode_READ(kind, str, idx + 7) == 't' && - PyUnicode_READ(kind, str, idx + 8) == 'y') { - rval = _parse_constant(s, pystr, JSON_NegInfinity, idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - default: - fallthrough = 1; - } - /* Didn't find a string, object, array, or named constant. Look for a number. */ - if (fallthrough) - rval = _match_number_unicode(s, pystr, idx, next_idx_ptr); - return rval; -} - -static PyObject * -scanner_call(PyObject *self, PyObject *args, PyObject *kwds) -{ - /* Python callable interface to scan_once_{str,unicode} */ - PyObject *pystr; - PyObject *rval; - Py_ssize_t idx; - Py_ssize_t next_idx = -1; - static char *kwlist[] = {"string", "idx", NULL}; - PyScannerObject *s; - assert(PyScanner_Check(self)); - s = (PyScannerObject *)self; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&:scan_once", kwlist, &pystr, _convertPyInt_AsSsize_t, &idx)) - return NULL; - - if (PyUnicode_Check(pystr)) { - if (PyUnicode_READY(pystr)) - return NULL; - rval = scan_once_unicode(s, pystr, idx, &next_idx); - } -#if PY_MAJOR_VERSION < 3 - else if (PyString_Check(pystr)) { - rval = scan_once_str(s, pystr, idx, &next_idx); - } -#endif /* PY_MAJOR_VERSION < 3 */ - else { - PyErr_Format(PyExc_TypeError, - "first argument must be a string, not %.80s", - Py_TYPE(pystr)->tp_name); - return NULL; - } - PyDict_Clear(s->memo); - return _build_rval_index_tuple(rval, next_idx); -} - -static PyObject * -JSON_ParseEncoding(PyObject *encoding) -{ - if (encoding == Py_None) - return JSON_InternFromString(DEFAULT_ENCODING); -#if PY_MAJOR_VERSION >= 3 - if (PyUnicode_Check(encoding)) { - if (PyUnicode_AsUTF8(encoding) == NULL) { - return NULL; - } - Py_INCREF(encoding); - return encoding; - } -#else /* PY_MAJOR_VERSION >= 3 */ - if (PyString_Check(encoding)) { - Py_INCREF(encoding); - return encoding; - } - if (PyUnicode_Check(encoding)) - return PyUnicode_AsEncodedString(encoding, NULL, NULL); -#endif /* PY_MAJOR_VERSION >= 3 */ - PyErr_SetString(PyExc_TypeError, "encoding must be a string"); - return NULL; -} - -static PyObject * -scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - /* Initialize Scanner object */ - PyObject *ctx; - static char *kwlist[] = {"context", NULL}; - PyScannerObject *s; - PyObject *encoding; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:make_scanner", kwlist, &ctx)) - return NULL; - - s = (PyScannerObject *)type->tp_alloc(type, 0); - if (s == NULL) - return NULL; - - if (s->memo == NULL) { - s->memo = PyDict_New(); - if (s->memo == NULL) - goto bail; - } - - encoding = PyObject_GetAttrString(ctx, "encoding"); - if (encoding == NULL) - goto bail; - s->encoding = JSON_ParseEncoding(encoding); - Py_XDECREF(encoding); - if (s->encoding == NULL) - goto bail; - - /* All of these will fail "gracefully" so we don't need to verify them */ - s->strict_bool = PyObject_GetAttrString(ctx, "strict"); - if (s->strict_bool == NULL) - goto bail; - s->strict = PyObject_IsTrue(s->strict_bool); - if (s->strict < 0) - goto bail; - s->object_hook = PyObject_GetAttrString(ctx, "object_hook"); - if (s->object_hook == NULL) - goto bail; - s->pairs_hook = PyObject_GetAttrString(ctx, "object_pairs_hook"); - if (s->pairs_hook == NULL) - goto bail; - s->parse_float = PyObject_GetAttrString(ctx, "parse_float"); - if (s->parse_float == NULL) - goto bail; - s->parse_int = PyObject_GetAttrString(ctx, "parse_int"); - if (s->parse_int == NULL) - goto bail; - s->parse_constant = PyObject_GetAttrString(ctx, "parse_constant"); - if (s->parse_constant == NULL) - goto bail; - - return (PyObject *)s; - -bail: - Py_DECREF(s); - return NULL; -} - -PyDoc_STRVAR(scanner_doc, "JSON scanner object"); - -static -PyTypeObject PyScannerType = { - PyVarObject_HEAD_INIT(NULL, 0) - "simplejson._speedups.Scanner", /* tp_name */ - sizeof(PyScannerObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - scanner_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - scanner_call, /* tp_call */ - 0, /* tp_str */ - 0,/* PyObject_GenericGetAttr, */ /* tp_getattro */ - 0,/* PyObject_GenericSetAttr, */ /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - scanner_doc, /* tp_doc */ - scanner_traverse, /* tp_traverse */ - scanner_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - scanner_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0,/* PyType_GenericAlloc, */ /* tp_alloc */ - scanner_new, /* tp_new */ - 0,/* PyObject_GC_Del, */ /* tp_free */ -}; - -static PyObject * -encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = { - "markers", - "default", - "encoder", - "indent", - "key_separator", - "item_separator", - "sort_keys", - "skipkeys", - "allow_nan", - "key_memo", - "use_decimal", - "namedtuple_as_object", - "tuple_as_array", - "int_as_string_bitcount", - "item_sort_key", - "encoding", - "for_json", - "ignore_nan", - "Decimal", - "iterable_as_array", - NULL}; - - PyEncoderObject *s; - PyObject *markers, *defaultfn, *encoder, *indent, *key_separator; - PyObject *item_separator, *sort_keys, *skipkeys, *allow_nan, *key_memo; - PyObject *use_decimal, *namedtuple_as_object, *tuple_as_array, *iterable_as_array; - PyObject *int_as_string_bitcount, *item_sort_key, *encoding, *for_json; - PyObject *ignore_nan, *Decimal; - int is_true; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOOOOOOOOOOOOOOOOOOO:make_encoder", kwlist, - &markers, &defaultfn, &encoder, &indent, &key_separator, &item_separator, - &sort_keys, &skipkeys, &allow_nan, &key_memo, &use_decimal, - &namedtuple_as_object, &tuple_as_array, - &int_as_string_bitcount, &item_sort_key, &encoding, &for_json, - &ignore_nan, &Decimal, &iterable_as_array)) - return NULL; - - s = (PyEncoderObject *)type->tp_alloc(type, 0); - if (s == NULL) - return NULL; - - Py_INCREF(markers); - s->markers = markers; - Py_INCREF(defaultfn); - s->defaultfn = defaultfn; - Py_INCREF(encoder); - s->encoder = encoder; -#if PY_MAJOR_VERSION >= 3 - if (encoding == Py_None) { - s->encoding = NULL; - } - else -#endif /* PY_MAJOR_VERSION >= 3 */ - { - s->encoding = JSON_ParseEncoding(encoding); - if (s->encoding == NULL) - goto bail; - } - Py_INCREF(indent); - s->indent = indent; - Py_INCREF(key_separator); - s->key_separator = key_separator; - Py_INCREF(item_separator); - s->item_separator = item_separator; - Py_INCREF(skipkeys); - s->skipkeys_bool = skipkeys; - s->skipkeys = PyObject_IsTrue(skipkeys); - if (s->skipkeys < 0) - goto bail; - Py_INCREF(key_memo); - s->key_memo = key_memo; - s->fast_encode = (PyCFunction_Check(s->encoder) && PyCFunction_GetFunction(s->encoder) == (PyCFunction)py_encode_basestring_ascii); - is_true = PyObject_IsTrue(ignore_nan); - if (is_true < 0) - goto bail; - s->allow_or_ignore_nan = is_true ? JSON_IGNORE_NAN : 0; - is_true = PyObject_IsTrue(allow_nan); - if (is_true < 0) - goto bail; - s->allow_or_ignore_nan |= is_true ? JSON_ALLOW_NAN : 0; - s->use_decimal = PyObject_IsTrue(use_decimal); - if (s->use_decimal < 0) - goto bail; - s->namedtuple_as_object = PyObject_IsTrue(namedtuple_as_object); - if (s->namedtuple_as_object < 0) - goto bail; - s->tuple_as_array = PyObject_IsTrue(tuple_as_array); - if (s->tuple_as_array < 0) - goto bail; - s->iterable_as_array = PyObject_IsTrue(iterable_as_array); - if (s->iterable_as_array < 0) - goto bail; - if (PyInt_Check(int_as_string_bitcount) || PyLong_Check(int_as_string_bitcount)) { - static const unsigned long long_long_bitsize = SIZEOF_LONG_LONG * 8; - long int_as_string_bitcount_val = PyLong_AsLong(int_as_string_bitcount); - if (int_as_string_bitcount_val > 0 && int_as_string_bitcount_val < (long)long_long_bitsize) { - s->max_long_size = PyLong_FromUnsignedLongLong(1ULL << (int)int_as_string_bitcount_val); - s->min_long_size = PyLong_FromLongLong(-1LL << (int)int_as_string_bitcount_val); - if (s->min_long_size == NULL || s->max_long_size == NULL) { - goto bail; - } - } - else { - PyErr_Format(PyExc_TypeError, - "int_as_string_bitcount (%ld) must be greater than 0 and less than the number of bits of a `long long` type (%lu bits)", - int_as_string_bitcount_val, long_long_bitsize); - goto bail; - } - } - else if (int_as_string_bitcount == Py_None) { - Py_INCREF(Py_None); - s->max_long_size = Py_None; - Py_INCREF(Py_None); - s->min_long_size = Py_None; - } - else { - PyErr_SetString(PyExc_TypeError, "int_as_string_bitcount must be None or an integer"); - goto bail; - } - if (item_sort_key != Py_None) { - if (!PyCallable_Check(item_sort_key)) { - PyErr_SetString(PyExc_TypeError, "item_sort_key must be None or callable"); - goto bail; - } - } - else { - is_true = PyObject_IsTrue(sort_keys); - if (is_true < 0) - goto bail; - if (is_true) { - static PyObject *itemgetter0 = NULL; - if (!itemgetter0) { - PyObject *operator = PyImport_ImportModule("operator"); - if (!operator) - goto bail; - itemgetter0 = PyObject_CallMethod(operator, "itemgetter", "i", 0); - Py_DECREF(operator); - } - item_sort_key = itemgetter0; - if (!item_sort_key) - goto bail; - } - } - if (item_sort_key == Py_None) { - Py_INCREF(Py_None); - s->item_sort_kw = Py_None; - } - else { - s->item_sort_kw = PyDict_New(); - if (s->item_sort_kw == NULL) - goto bail; - if (PyDict_SetItemString(s->item_sort_kw, "key", item_sort_key)) - goto bail; - } - Py_INCREF(sort_keys); - s->sort_keys = sort_keys; - Py_INCREF(item_sort_key); - s->item_sort_key = item_sort_key; - Py_INCREF(Decimal); - s->Decimal = Decimal; - s->for_json = PyObject_IsTrue(for_json); - if (s->for_json < 0) - goto bail; - - return (PyObject *)s; - -bail: - Py_DECREF(s); - return NULL; -} - -static PyObject * -encoder_call(PyObject *self, PyObject *args, PyObject *kwds) -{ - /* Python callable interface to encode_listencode_obj */ - static char *kwlist[] = {"obj", "_current_indent_level", NULL}; - PyObject *obj; - Py_ssize_t indent_level; - PyEncoderObject *s; - JSON_Accu rval; - assert(PyEncoder_Check(self)); - s = (PyEncoderObject *)self; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&:_iterencode", kwlist, - &obj, _convertPyInt_AsSsize_t, &indent_level)) - return NULL; - if (JSON_Accu_Init(&rval)) - return NULL; - if (encoder_listencode_obj(s, &rval, obj, indent_level)) { - JSON_Accu_Destroy(&rval); - return NULL; - } - return JSON_Accu_FinishAsList(&rval); -} - -static PyObject * -_encoded_const(PyObject *obj) -{ - /* Return the JSON string representation of None, True, False */ - if (obj == Py_None) { - static PyObject *s_null = NULL; - if (s_null == NULL) { - s_null = JSON_InternFromString("null"); - } - Py_INCREF(s_null); - return s_null; - } - else if (obj == Py_True) { - static PyObject *s_true = NULL; - if (s_true == NULL) { - s_true = JSON_InternFromString("true"); - } - Py_INCREF(s_true); - return s_true; - } - else if (obj == Py_False) { - static PyObject *s_false = NULL; - if (s_false == NULL) { - s_false = JSON_InternFromString("false"); - } - Py_INCREF(s_false); - return s_false; - } - else { - PyErr_SetString(PyExc_ValueError, "not a const"); - return NULL; - } -} - -static PyObject * -encoder_encode_float(PyEncoderObject *s, PyObject *obj) -{ - /* Return the JSON representation of a PyFloat */ - double i = PyFloat_AS_DOUBLE(obj); - if (!Py_IS_FINITE(i)) { - if (!s->allow_or_ignore_nan) { - PyErr_SetString(PyExc_ValueError, "Out of range float values are not JSON compliant"); - return NULL; - } - if (s->allow_or_ignore_nan & JSON_IGNORE_NAN) { - return _encoded_const(Py_None); - } - /* JSON_ALLOW_NAN is set */ - else if (i > 0) { - Py_INCREF(JSON_Infinity); - return JSON_Infinity; - } - else if (i < 0) { - Py_INCREF(JSON_NegInfinity); - return JSON_NegInfinity; - } - else { - Py_INCREF(JSON_NaN); - return JSON_NaN; - } - } - /* Use a better float format here? */ - if (PyFloat_CheckExact(obj)) { - return PyObject_Repr(obj); - } - else { - /* See #118, do not trust custom str/repr */ - PyObject *res; - PyObject *tmp = PyObject_CallOneArg((PyObject *)&PyFloat_Type, obj); - if (tmp == NULL) { - return NULL; - } - res = PyObject_Repr(tmp); - Py_DECREF(tmp); - return res; - } -} - -static PyObject * -encoder_encode_string(PyEncoderObject *s, PyObject *obj) -{ - /* Return the JSON representation of a string */ - PyObject *encoded; - - if (s->fast_encode) { - return py_encode_basestring_ascii(NULL, obj); - } - encoded = PyObject_CallOneArg(s->encoder, obj); - if (encoded != NULL && -#if PY_MAJOR_VERSION < 3 - !PyString_Check(encoded) && -#endif /* PY_MAJOR_VERSION < 3 */ - !PyUnicode_Check(encoded)) - { - PyErr_Format(PyExc_TypeError, - "encoder() must return a string, not %.80s", - Py_TYPE(encoded)->tp_name); - Py_DECREF(encoded); - return NULL; - } - return encoded; -} - -static int -_steal_accumulate(JSON_Accu *accu, PyObject *stolen) -{ - /* Append stolen and then decrement its reference count */ - int rval = JSON_Accu_Accumulate(accu, stolen); - Py_DECREF(stolen); - return rval; -} - -static int -encoder_listencode_obj(PyEncoderObject *s, JSON_Accu *rval, PyObject *obj, Py_ssize_t indent_level) -{ - /* Encode Python object obj to a JSON term, rval is a PyList */ - int rv = -1; - do { - PyObject *newobj; - if (obj == Py_None || obj == Py_True || obj == Py_False) { - PyObject *cstr = _encoded_const(obj); - if (cstr != NULL) - rv = _steal_accumulate(rval, cstr); - } - else if ((PyBytes_Check(obj) && s->encoding != NULL) || - PyUnicode_Check(obj)) - { - PyObject *encoded = encoder_encode_string(s, obj); - if (encoded != NULL) - rv = _steal_accumulate(rval, encoded); - } - else if (PyInt_Check(obj) || PyLong_Check(obj)) { - PyObject *encoded; - if (PyInt_CheckExact(obj) || PyLong_CheckExact(obj)) { - encoded = PyObject_Str(obj); - } - else { - /* See #118, do not trust custom str/repr */ - PyObject *tmp = PyObject_CallOneArg((PyObject *)&PyLong_Type, obj); - if (tmp == NULL) { - encoded = NULL; - } - else { - encoded = PyObject_Str(tmp); - Py_DECREF(tmp); - } - } - if (encoded != NULL) { - encoded = maybe_quote_bigint(s, encoded, obj); - if (encoded == NULL) - break; - rv = _steal_accumulate(rval, encoded); - } - } - else if (PyFloat_Check(obj)) { - PyObject *encoded = encoder_encode_float(s, obj); - if (encoded != NULL) - rv = _steal_accumulate(rval, encoded); - } - else if (s->for_json && _call_json_method(obj, FOR_JSON_METHOD_NAME, &newobj)) { - if (newobj == NULL) { - return -1; - } - if (Py_EnterRecursiveCall(" while encoding a JSON object")) { - Py_DECREF(newobj); - return rv; - } - rv = encoder_listencode_obj(s, rval, newobj, indent_level); - Py_DECREF(newobj); - Py_LeaveRecursiveCall(); - } - else if (s->namedtuple_as_object && _call_json_method(obj, ASDICT_METHOD_NAME, &newobj)) { - if (newobj == NULL) { - return -1; - } - if (Py_EnterRecursiveCall(" while encoding a JSON object")) { - Py_DECREF(newobj); - return rv; - } - if (PyDict_Check(newobj)) { - rv = encoder_listencode_dict(s, rval, newobj, indent_level); - } else { - PyErr_Format( - PyExc_TypeError, - "_asdict() must return a dict, not %.80s", - Py_TYPE(newobj)->tp_name - ); - rv = -1; - } - Py_DECREF(newobj); - Py_LeaveRecursiveCall(); - } - else if (PyList_Check(obj) || (s->tuple_as_array && PyTuple_Check(obj))) { - if (Py_EnterRecursiveCall(" while encoding a JSON object")) - return rv; - rv = encoder_listencode_list(s, rval, obj, indent_level); - Py_LeaveRecursiveCall(); - } - else if (PyDict_Check(obj)) { - if (Py_EnterRecursiveCall(" while encoding a JSON object")) - return rv; - rv = encoder_listencode_dict(s, rval, obj, indent_level); - Py_LeaveRecursiveCall(); - } - else if (s->use_decimal && PyObject_TypeCheck(obj, (PyTypeObject *)s->Decimal)) { - PyObject *encoded = PyObject_Str(obj); - if (encoded != NULL) - rv = _steal_accumulate(rval, encoded); - } - else if (is_raw_json(obj)) - { - PyObject *encoded = PyObject_GetAttrString(obj, "encoded_json"); - if (encoded != NULL) - rv = _steal_accumulate(rval, encoded); - } - else { - PyObject *ident = NULL; - PyObject *newobj; - if (s->iterable_as_array) { - newobj = PyObject_GetIter(obj); - if (newobj == NULL) - PyErr_Clear(); - else { - rv = encoder_listencode_list(s, rval, newobj, indent_level); - Py_DECREF(newobj); - break; - } - } - if (s->markers != Py_None) { - int has_key; - ident = PyLong_FromVoidPtr(obj); - if (ident == NULL) - break; - has_key = PyDict_Contains(s->markers, ident); - if (has_key) { - if (has_key != -1) - PyErr_SetString(PyExc_ValueError, "Circular reference detected"); - Py_DECREF(ident); - break; - } - if (PyDict_SetItem(s->markers, ident, obj)) { - Py_DECREF(ident); - break; - } - } - if (Py_EnterRecursiveCall(" while encoding a JSON object")) - return rv; - newobj = PyObject_CallOneArg(s->defaultfn, obj); - if (newobj == NULL) { - Py_XDECREF(ident); - Py_LeaveRecursiveCall(); - break; - } - rv = encoder_listencode_obj(s, rval, newobj, indent_level); - Py_LeaveRecursiveCall(); - Py_DECREF(newobj); - if (rv) { - Py_XDECREF(ident); - rv = -1; - } - else if (ident != NULL) { - if (PyDict_DelItem(s->markers, ident)) { - Py_XDECREF(ident); - rv = -1; - } - Py_XDECREF(ident); - } - } - } while (0); - return rv; -} - -static int -encoder_listencode_dict(PyEncoderObject *s, JSON_Accu *rval, PyObject *dct, Py_ssize_t indent_level) -{ - /* Encode Python dict dct a JSON term */ - static PyObject *open_dict = NULL; - static PyObject *close_dict = NULL; - static PyObject *empty_dict = NULL; - PyObject *kstr = NULL; - PyObject *ident = NULL; - PyObject *iter = NULL; - PyObject *item = NULL; - PyObject *items = NULL; - PyObject *encoded = NULL; - Py_ssize_t idx; - - if (open_dict == NULL || close_dict == NULL || empty_dict == NULL) { - open_dict = JSON_InternFromString("{"); - close_dict = JSON_InternFromString("}"); - empty_dict = JSON_InternFromString("{}"); - if (open_dict == NULL || close_dict == NULL || empty_dict == NULL) - return -1; - } - if (PyDict_Size(dct) == 0) - return JSON_Accu_Accumulate(rval, empty_dict); - - if (s->markers != Py_None) { - int has_key; - ident = PyLong_FromVoidPtr(dct); - if (ident == NULL) - goto bail; - has_key = PyDict_Contains(s->markers, ident); - if (has_key) { - if (has_key != -1) - PyErr_SetString(PyExc_ValueError, "Circular reference detected"); - goto bail; - } - if (PyDict_SetItem(s->markers, ident, dct)) { - goto bail; - } - } - - if (JSON_Accu_Accumulate(rval, open_dict)) - goto bail; - - if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level += 1; - /* - newline_indent = '\n' + (_indent * _current_indent_level) - separator = _item_separator + newline_indent - buf += newline_indent - */ - } - - iter = encoder_dict_iteritems(s, dct); - if (iter == NULL) - goto bail; - - idx = 0; - while ((item = PyIter_Next(iter))) { - PyObject *encoded, *key, *value; - if (!PyTuple_Check(item) || Py_SIZE(item) != 2) { - PyErr_SetString(PyExc_ValueError, "items must return 2-tuples"); - goto bail; - } - key = PyTuple_GET_ITEM(item, 0); - if (key == NULL) - goto bail; - value = PyTuple_GET_ITEM(item, 1); - if (value == NULL) - goto bail; - - kstr = encoder_stringify_key(s, key); - if (kstr == NULL) - goto bail; - else if (kstr == Py_None) { - /* skipkeys */ - Py_DECREF(item); - Py_DECREF(kstr); - continue; - } - if (idx) { - if (JSON_Accu_Accumulate(rval, s->item_separator)) - goto bail; - } - /* - * Only cache the encoding of string keys. False and True are - * indistinguishable from 0 and 1 in a dictionary lookup and there - * may be other quirks with user defined subclasses. - */ - encoded = PyDict_GetItem(s->key_memo, kstr); - if (encoded != NULL) { - Py_INCREF(encoded); - Py_CLEAR(kstr); - } else { - encoded = encoder_encode_string(s, kstr); - Py_CLEAR(kstr); - if (encoded == NULL) - goto bail; - if (PyDict_SetItem(s->key_memo, key, encoded)) - goto bail; - } - if (JSON_Accu_Accumulate(rval, encoded)) { - goto bail; - } - Py_CLEAR(encoded); - if (JSON_Accu_Accumulate(rval, s->key_separator)) - goto bail; - if (encoder_listencode_obj(s, rval, value, indent_level)) - goto bail; - Py_CLEAR(item); - idx += 1; - } - Py_CLEAR(iter); - if (PyErr_Occurred()) - goto bail; - if (ident != NULL) { - if (PyDict_DelItem(s->markers, ident)) - goto bail; - Py_CLEAR(ident); - } - if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level -= 1; - /* - yield '\n' + (_indent * _current_indent_level) - */ - } - if (JSON_Accu_Accumulate(rval, close_dict)) - goto bail; - return 0; - -bail: - Py_XDECREF(encoded); - Py_XDECREF(items); - Py_XDECREF(item); - Py_XDECREF(iter); - Py_XDECREF(kstr); - Py_XDECREF(ident); - return -1; -} - - -static int -encoder_listencode_list(PyEncoderObject *s, JSON_Accu *rval, PyObject *seq, Py_ssize_t indent_level) -{ - /* Encode Python list seq to a JSON term */ - static PyObject *open_array = NULL; - static PyObject *close_array = NULL; - static PyObject *empty_array = NULL; - PyObject *ident = NULL; - PyObject *iter = NULL; - PyObject *obj = NULL; - int is_true; - int i = 0; - - if (open_array == NULL || close_array == NULL || empty_array == NULL) { - open_array = JSON_InternFromString("["); - close_array = JSON_InternFromString("]"); - empty_array = JSON_InternFromString("[]"); - if (open_array == NULL || close_array == NULL || empty_array == NULL) - return -1; - } - ident = NULL; - is_true = PyObject_IsTrue(seq); - if (is_true == -1) - return -1; - else if (is_true == 0) - return JSON_Accu_Accumulate(rval, empty_array); - - if (s->markers != Py_None) { - int has_key; - ident = PyLong_FromVoidPtr(seq); - if (ident == NULL) - goto bail; - has_key = PyDict_Contains(s->markers, ident); - if (has_key) { - if (has_key != -1) - PyErr_SetString(PyExc_ValueError, "Circular reference detected"); - goto bail; - } - if (PyDict_SetItem(s->markers, ident, seq)) { - goto bail; - } - } - - iter = PyObject_GetIter(seq); - if (iter == NULL) - goto bail; - - if (JSON_Accu_Accumulate(rval, open_array)) - goto bail; - if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level += 1; - /* - newline_indent = '\n' + (_indent * _current_indent_level) - separator = _item_separator + newline_indent - buf += newline_indent - */ - } - while ((obj = PyIter_Next(iter))) { - if (i) { - if (JSON_Accu_Accumulate(rval, s->item_separator)) - goto bail; - } - if (encoder_listencode_obj(s, rval, obj, indent_level)) - goto bail; - i++; - Py_CLEAR(obj); - } - Py_CLEAR(iter); - if (PyErr_Occurred()) - goto bail; - if (ident != NULL) { - if (PyDict_DelItem(s->markers, ident)) - goto bail; - Py_CLEAR(ident); - } - if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level -= 1; - /* - yield '\n' + (_indent * _current_indent_level) - */ - } - if (JSON_Accu_Accumulate(rval, close_array)) - goto bail; - return 0; - -bail: - Py_XDECREF(obj); - Py_XDECREF(iter); - Py_XDECREF(ident); - return -1; -} - -static void -encoder_dealloc(PyObject *self) -{ - /* bpo-31095: UnTrack is needed before calling any callbacks */ - PyObject_GC_UnTrack(self); - encoder_clear(self); - Py_TYPE(self)->tp_free(self); -} - -static int -encoder_traverse(PyObject *self, visitproc visit, void *arg) -{ - PyEncoderObject *s; - assert(PyEncoder_Check(self)); - s = (PyEncoderObject *)self; - Py_VISIT(s->markers); - Py_VISIT(s->defaultfn); - Py_VISIT(s->encoder); - Py_VISIT(s->encoding); - Py_VISIT(s->indent); - Py_VISIT(s->key_separator); - Py_VISIT(s->item_separator); - Py_VISIT(s->key_memo); - Py_VISIT(s->sort_keys); - Py_VISIT(s->item_sort_kw); - Py_VISIT(s->item_sort_key); - Py_VISIT(s->max_long_size); - Py_VISIT(s->min_long_size); - Py_VISIT(s->Decimal); - return 0; -} - -static int -encoder_clear(PyObject *self) -{ - /* Deallocate Encoder */ - PyEncoderObject *s; - assert(PyEncoder_Check(self)); - s = (PyEncoderObject *)self; - Py_CLEAR(s->markers); - Py_CLEAR(s->defaultfn); - Py_CLEAR(s->encoder); - Py_CLEAR(s->encoding); - Py_CLEAR(s->indent); - Py_CLEAR(s->key_separator); - Py_CLEAR(s->item_separator); - Py_CLEAR(s->key_memo); - Py_CLEAR(s->skipkeys_bool); - Py_CLEAR(s->sort_keys); - Py_CLEAR(s->item_sort_kw); - Py_CLEAR(s->item_sort_key); - Py_CLEAR(s->max_long_size); - Py_CLEAR(s->min_long_size); - Py_CLEAR(s->Decimal); - return 0; -} - -PyDoc_STRVAR(encoder_doc, "_iterencode(obj, _current_indent_level) -> iterable"); - -static -PyTypeObject PyEncoderType = { - PyVarObject_HEAD_INIT(NULL, 0) - "simplejson._speedups.Encoder", /* tp_name */ - sizeof(PyEncoderObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - encoder_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - encoder_call, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - encoder_doc, /* tp_doc */ - encoder_traverse, /* tp_traverse */ - encoder_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - encoder_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - encoder_new, /* tp_new */ - 0, /* tp_free */ -}; - -static PyMethodDef speedups_methods[] = { - {"encode_basestring_ascii", - (PyCFunction)py_encode_basestring_ascii, - METH_O, - pydoc_encode_basestring_ascii}, - {"scanstring", - (PyCFunction)py_scanstring, - METH_VARARGS, - pydoc_scanstring}, - {NULL, NULL, 0, NULL} -}; - -PyDoc_STRVAR(module_doc, -"simplejson speedups\n"); - -#if PY_MAJOR_VERSION >= 3 -static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - "_speedups", /* m_name */ - module_doc, /* m_doc */ - -1, /* m_size */ - speedups_methods, /* m_methods */ - NULL, /* m_reload */ - NULL, /* m_traverse */ - NULL, /* m_clear*/ - NULL, /* m_free */ -}; -#endif - -PyObject * -import_dependency(char *module_name, char *attr_name) -{ - PyObject *rval; - PyObject *module = PyImport_ImportModule(module_name); - if (module == NULL) - return NULL; - rval = PyObject_GetAttrString(module, attr_name); - Py_DECREF(module); - return rval; -} - -static int -init_constants(void) -{ - JSON_NaN = JSON_InternFromString("NaN"); - if (JSON_NaN == NULL) - return 0; - JSON_Infinity = JSON_InternFromString("Infinity"); - if (JSON_Infinity == NULL) - return 0; - JSON_NegInfinity = JSON_InternFromString("-Infinity"); - if (JSON_NegInfinity == NULL) - return 0; -#if PY_MAJOR_VERSION >= 3 - JSON_EmptyUnicode = PyUnicode_New(0, 127); -#else /* PY_MAJOR_VERSION >= 3 */ - JSON_EmptyStr = PyString_FromString(""); - if (JSON_EmptyStr == NULL) - return 0; - JSON_EmptyUnicode = PyUnicode_FromUnicode(NULL, 0); -#endif /* PY_MAJOR_VERSION >= 3 */ - if (JSON_EmptyUnicode == NULL) - return 0; - - return 1; -} - -static PyObject * -moduleinit(void) -{ - PyObject *m; - if (PyType_Ready(&PyScannerType) < 0) - return NULL; - if (PyType_Ready(&PyEncoderType) < 0) - return NULL; - if (!init_constants()) - return NULL; - -#if PY_MAJOR_VERSION >= 3 - m = PyModule_Create(&moduledef); -#else - m = Py_InitModule3("_speedups", speedups_methods, module_doc); -#endif - Py_INCREF((PyObject*)&PyScannerType); - PyModule_AddObject(m, "make_scanner", (PyObject*)&PyScannerType); - Py_INCREF((PyObject*)&PyEncoderType); - PyModule_AddObject(m, "make_encoder", (PyObject*)&PyEncoderType); - RawJSONType = import_dependency("simplejson.raw_json", "RawJSON"); - if (RawJSONType == NULL) - return NULL; - JSONDecodeError = import_dependency("simplejson.errors", "JSONDecodeError"); - if (JSONDecodeError == NULL) - return NULL; - return m; -} - -#if PY_MAJOR_VERSION >= 3 -PyMODINIT_FUNC -PyInit__speedups(void) -{ - return moduleinit(); -} -#else -void -init_speedups(void) -{ - moduleinit(); -} -#endif diff --git a/contrib/python/simplejson/py3/simplejson/compat.py b/contrib/python/simplejson/py3/simplejson/compat.py deleted file mode 100644 index 5fc14128443..00000000000 --- a/contrib/python/simplejson/py3/simplejson/compat.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Python 3 compatibility shims -""" -import sys -if sys.version_info[0] < 3: - PY3 = False - def b(s): - return s - try: - from cStringIO import StringIO - except ImportError: - from StringIO import StringIO - BytesIO = StringIO - text_type = unicode - binary_type = str - string_types = (basestring,) - integer_types = (int, long) - unichr = unichr - reload_module = reload -else: - PY3 = True - if sys.version_info[:2] >= (3, 4): - from importlib import reload as reload_module - else: - from imp import reload as reload_module - def b(s): - return bytes(s, 'latin1') - from io import StringIO, BytesIO - text_type = str - binary_type = bytes - string_types = (str,) - integer_types = (int,) - unichr = chr - -long_type = integer_types[-1] diff --git a/contrib/python/simplejson/py3/simplejson/decoder.py b/contrib/python/simplejson/py3/simplejson/decoder.py deleted file mode 100644 index c99a976d84e..00000000000 --- a/contrib/python/simplejson/py3/simplejson/decoder.py +++ /dev/null @@ -1,416 +0,0 @@ -"""Implementation of JSONDecoder -""" -from __future__ import absolute_import -import re -import sys -import struct -from .compat import PY3, unichr -from .scanner import make_scanner, JSONDecodeError - -def _import_c_scanstring(): - try: - from ._speedups import scanstring - return scanstring - except ImportError: - return None -c_scanstring = _import_c_scanstring() - -# NOTE (3.1.0): JSONDecodeError may still be imported from this module for -# compatibility, but it was never in the __all__ -__all__ = ['JSONDecoder'] - -FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL - -def _floatconstants(): - if sys.version_info < (2, 6): - _BYTES = '7FF80000000000007FF0000000000000'.decode('hex') - nan, inf = struct.unpack('>dd', _BYTES) - else: - nan = float('nan') - inf = float('inf') - return nan, inf, -inf - -NaN, PosInf, NegInf = _floatconstants() - -_CONSTANTS = { - '-Infinity': NegInf, - 'Infinity': PosInf, - 'NaN': NaN, -} - -STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS) -BACKSLASH = { - '"': u'"', '\\': u'\\', '/': u'/', - 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t', -} - -DEFAULT_ENCODING = "utf-8" - -if hasattr(sys, 'get_int_max_str_digits'): - bounded_int = int -else: - def bounded_int(s, INT_MAX_STR_DIGITS=4300): - """Backport of the integer string length conversion limitation - - https://docs.python.org/3/library/stdtypes.html#int-max-str-digits - """ - if len(s) > INT_MAX_STR_DIGITS: - raise ValueError("Exceeds the limit (%s) for integer string conversion: value has %s digits" % (INT_MAX_STR_DIGITS, len(s))) - return int(s) - - -def scan_four_digit_hex(s, end, _m=re.compile(r'^[0-9a-fA-F]{4}$').match): - """Scan a four digit hex number from s[end:end + 4] - """ - msg = "Invalid \\uXXXX escape sequence" - esc = s[end:end + 4] - if not _m(esc): - raise JSONDecodeError(msg, s, end - 2) - try: - return int(esc, 16), end + 4 - except ValueError: - raise JSONDecodeError(msg, s, end - 2) - -def py_scanstring(s, end, encoding=None, strict=True, - _b=BACKSLASH, _m=STRINGCHUNK.match, _join=u''.join, - _PY3=PY3, _maxunicode=sys.maxunicode, - _scan_four_digit_hex=scan_four_digit_hex): - """Scan the string s for a JSON string. End is the index of the - character in s after the quote that started the JSON string. - Unescapes all valid JSON string escape sequences and raises ValueError - on attempt to decode an invalid string. If strict is False then literal - control characters are allowed in the string. - - Returns a tuple of the decoded string and the index of the character in s - after the end quote.""" - if encoding is None: - encoding = DEFAULT_ENCODING - chunks = [] - _append = chunks.append - begin = end - 1 - while 1: - chunk = _m(s, end) - if chunk is None: - raise JSONDecodeError( - "Unterminated string starting at", s, begin) - prev_end = end - end = chunk.end() - content, terminator = chunk.groups() - # Content is contains zero or more unescaped string characters - if content: - if not _PY3 and not isinstance(content, unicode): - content = unicode(content, encoding) - _append(content) - # Terminator is the end of string, a literal control character, - # or a backslash denoting that an escape sequence follows - if terminator == '"': - break - elif terminator != '\\': - if strict: - msg = "Invalid control character %r at" - raise JSONDecodeError(msg, s, prev_end) - else: - _append(terminator) - continue - try: - esc = s[end] - except IndexError: - raise JSONDecodeError( - "Unterminated string starting at", s, begin) - # If not a unicode escape sequence, must be in the lookup table - if esc != 'u': - try: - char = _b[esc] - except KeyError: - msg = "Invalid \\X escape sequence %r" - raise JSONDecodeError(msg, s, end) - end += 1 - else: - # Unicode escape sequence - uni, end = _scan_four_digit_hex(s, end + 1) - # Check for surrogate pair on UCS-4 systems - # Note that this will join high/low surrogate pairs - # but will also pass unpaired surrogates through - if (_maxunicode > 65535 and - uni & 0xfc00 == 0xd800 and - s[end:end + 2] == '\\u'): - uni2, end2 = _scan_four_digit_hex(s, end + 2) - if uni2 & 0xfc00 == 0xdc00: - uni = 0x10000 + (((uni - 0xd800) << 10) | - (uni2 - 0xdc00)) - end = end2 - char = unichr(uni) - # Append the unescaped character - _append(char) - return _join(chunks), end - - -# Use speedup if available -scanstring = c_scanstring or py_scanstring - -WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS) -WHITESPACE_STR = ' \t\n\r' - -def JSONObject(state, encoding, strict, scan_once, object_hook, - object_pairs_hook, memo=None, - _w=WHITESPACE.match, _ws=WHITESPACE_STR): - (s, end) = state - # Backwards compatibility - if memo is None: - memo = {} - memo_get = memo.setdefault - pairs = [] - # Use a slice to prevent IndexError from being raised, the following - # check will raise a more specific ValueError if the string is empty - nextchar = s[end:end + 1] - # Normally we expect nextchar == '"' - if nextchar != '"': - if nextchar in _ws: - end = _w(s, end).end() - nextchar = s[end:end + 1] - # Trivial empty object - if nextchar == '}': - if object_pairs_hook is not None: - result = object_pairs_hook(pairs) - return result, end + 1 - pairs = {} - if object_hook is not None: - pairs = object_hook(pairs) - return pairs, end + 1 - elif nextchar != '"': - raise JSONDecodeError( - "Expecting property name enclosed in double quotes or '}'", - s, end) - end += 1 - while True: - key, end = scanstring(s, end, encoding, strict) - key = memo_get(key, key) - - # To skip some function call overhead we optimize the fast paths where - # the JSON key separator is ": " or just ":". - if s[end:end + 1] != ':': - end = _w(s, end).end() - if s[end:end + 1] != ':': - raise JSONDecodeError("Expecting ':' delimiter", s, end) - - end += 1 - - try: - if s[end] in _ws: - end += 1 - if s[end] in _ws: - end = _w(s, end + 1).end() - except IndexError: - pass - - value, end = scan_once(s, end) - pairs.append((key, value)) - - try: - nextchar = s[end] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end] - except IndexError: - nextchar = '' - end += 1 - - if nextchar == '}': - break - elif nextchar != ',': - raise JSONDecodeError("Expecting ',' delimiter or '}'", s, end - 1) - - try: - nextchar = s[end] - if nextchar in _ws: - end += 1 - nextchar = s[end] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end] - except IndexError: - nextchar = '' - - end += 1 - if nextchar != '"': - raise JSONDecodeError( - "Expecting property name enclosed in double quotes", - s, end - 1) - - if object_pairs_hook is not None: - result = object_pairs_hook(pairs) - return result, end - pairs = dict(pairs) - if object_hook is not None: - pairs = object_hook(pairs) - return pairs, end - -def JSONArray(state, scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): - (s, end) = state - values = [] - nextchar = s[end:end + 1] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end:end + 1] - # Look-ahead for trivial empty array - if nextchar == ']': - return values, end + 1 - elif nextchar == '': - raise JSONDecodeError("Expecting value or ']'", s, end) - _append = values.append - while True: - value, end = scan_once(s, end) - _append(value) - nextchar = s[end:end + 1] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end:end + 1] - end += 1 - if nextchar == ']': - break - elif nextchar != ',': - raise JSONDecodeError("Expecting ',' delimiter or ']'", s, end - 1) - - try: - if s[end] in _ws: - end += 1 - if s[end] in _ws: - end = _w(s, end + 1).end() - except IndexError: - pass - - return values, end - -class JSONDecoder(object): - """Simple JSON <http://json.org> decoder - - Performs the following translations in decoding by default: - - +---------------+-------------------+ - | JSON | Python | - +===============+===================+ - | object | dict | - +---------------+-------------------+ - | array | list | - +---------------+-------------------+ - | string | str, unicode | - +---------------+-------------------+ - | number (int) | int, long | - +---------------+-------------------+ - | number (real) | float | - +---------------+-------------------+ - | true | True | - +---------------+-------------------+ - | false | False | - +---------------+-------------------+ - | null | None | - +---------------+-------------------+ - - When allow_nan=True, it also understands - ``NaN``, ``Infinity``, and ``-Infinity`` as - their corresponding ``float`` values, which is outside the JSON spec. - - """ - - def __init__(self, encoding=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, strict=True, - object_pairs_hook=None, allow_nan=False): - """ - *encoding* determines the encoding used to interpret any - :class:`str` objects decoded by this instance (``'utf-8'`` by - default). It has no effect when decoding :class:`unicode` objects. - - Note that currently only encodings that are a superset of ASCII work, - strings of other encodings should be passed in as :class:`unicode`. - - *object_hook*, if specified, will be called with the result of every - JSON object decoded and its return value will be used in place of the - given :class:`dict`. This can be used to provide custom - deserializations (e.g. to support JSON-RPC class hinting). - - *object_pairs_hook* is an optional function that will be called with - the result of any object literal decode with an ordered list of pairs. - The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders - that rely on the order that the key and value pairs are decoded (for - example, :func:`collections.OrderedDict` will remember the order of - insertion). If *object_hook* is also defined, the *object_pairs_hook* - takes priority. - - *parse_float*, if specified, will be called with the string of every - JSON float to be decoded. By default, this is equivalent to - ``float(num_str)``. This can be used to use another datatype or parser - for JSON floats (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every - JSON int to be decoded. By default, this is equivalent to - ``int(num_str)``. This can be used to use another datatype or parser - for JSON integers (e.g. :class:`float`). - - *allow_nan*, if True (default false), will allow the parser to - accept the non-standard floats ``NaN``, ``Infinity``, and ``-Infinity``. - - *parse_constant*, if specified, will be - called with one of the following strings: ``'-Infinity'``, - ``'Infinity'``, ``'NaN'``. It is not recommended to use this feature, - as it is rare to parse non-compliant JSON containing these values. - - *strict* controls the parser's behavior when it encounters an - invalid control character in a string. The default setting of - ``True`` means that unescaped control characters are parse errors, if - ``False`` then control characters will be allowed in strings. - - """ - if encoding is None: - encoding = DEFAULT_ENCODING - self.encoding = encoding - self.object_hook = object_hook - self.object_pairs_hook = object_pairs_hook - self.parse_float = parse_float or float - self.parse_int = parse_int or bounded_int - self.parse_constant = parse_constant or (allow_nan and _CONSTANTS.__getitem__ or None) - self.strict = strict - self.parse_object = JSONObject - self.parse_array = JSONArray - self.parse_string = scanstring - self.memo = {} - self.scan_once = make_scanner(self) - - def decode(self, s, _w=WHITESPACE.match, _PY3=PY3): - """Return the Python representation of ``s`` (a ``str`` or ``unicode`` - instance containing a JSON document) - - """ - if _PY3 and isinstance(s, bytes): - s = str(s, self.encoding) - obj, end = self.raw_decode(s) - end = _w(s, end).end() - if end != len(s): - raise JSONDecodeError("Extra data", s, end, len(s)) - return obj - - def raw_decode(self, s, idx=0, _w=WHITESPACE.match, _PY3=PY3): - """Decode a JSON document from ``s`` (a ``str`` or ``unicode`` - beginning with a JSON document) and return a 2-tuple of the Python - representation and the index in ``s`` where the document ended. - Optionally, ``idx`` can be used to specify an offset in ``s`` where - the JSON document begins. - - This can be used to decode a JSON document from a string that may - have extraneous data at the end. - - """ - if idx < 0: - # Ensure that raw_decode bails on negative indexes, the regex - # would otherwise mask this behavior. #98 - raise JSONDecodeError('Expecting value', s, idx) - if _PY3 and not isinstance(s, str): - raise TypeError("Input string must be text, not bytes") - # strip UTF-8 bom - if len(s) > idx: - ord0 = ord(s[idx]) - if ord0 == 0xfeff: - idx += 1 - elif ord0 == 0xef and s[idx:idx + 3] == '\xef\xbb\xbf': - idx += 3 - return self.scan_once(s, idx=_w(s, idx).end()) diff --git a/contrib/python/simplejson/py3/simplejson/encoder.py b/contrib/python/simplejson/py3/simplejson/encoder.py deleted file mode 100644 index 661ff361bf2..00000000000 --- a/contrib/python/simplejson/py3/simplejson/encoder.py +++ /dev/null @@ -1,740 +0,0 @@ -"""Implementation of JSONEncoder -""" -from __future__ import absolute_import -import re -from operator import itemgetter -# Do not import Decimal directly to avoid reload issues -import decimal -from .compat import binary_type, text_type, string_types, integer_types, PY3 -def _import_speedups(): - try: - from . import _speedups - return _speedups.encode_basestring_ascii, _speedups.make_encoder - except ImportError: - return None, None -c_encode_basestring_ascii, c_make_encoder = _import_speedups() - -from .decoder import PosInf -from .raw_json import RawJSON - -ESCAPE = re.compile(r'[\x00-\x1f\\"]') -ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') -HAS_UTF8 = re.compile(r'[\x80-\xff]') -ESCAPE_DCT = { - '\\': '\\\\', - '"': '\\"', - '\b': '\\b', - '\f': '\\f', - '\n': '\\n', - '\r': '\\r', - '\t': '\\t', -} -for i in range(0x20): - #ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i)) - ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,)) -del i - -FLOAT_REPR = repr - -def encode_basestring(s, _PY3=PY3, _q=u'"'): - """Return a JSON representation of a Python string - - """ - if _PY3: - if isinstance(s, bytes): - s = str(s, 'utf-8') - elif type(s) is not str: - # convert an str subclass instance to exact str - # raise a TypeError otherwise - s = str.__str__(s) - else: - if isinstance(s, str) and HAS_UTF8.search(s) is not None: - s = unicode(s, 'utf-8') - elif type(s) not in (str, unicode): - # convert an str subclass instance to exact str - # convert a unicode subclass instance to exact unicode - # raise a TypeError otherwise - if isinstance(s, str): - s = str.__str__(s) - else: - s = unicode.__getnewargs__(s)[0] - def replace(match): - return ESCAPE_DCT[match.group(0)] - return _q + ESCAPE.sub(replace, s) + _q - - -def py_encode_basestring_ascii(s, _PY3=PY3): - """Return an ASCII-only JSON representation of a Python string - - """ - if _PY3: - if isinstance(s, bytes): - s = str(s, 'utf-8') - elif type(s) is not str: - # convert an str subclass instance to exact str - # raise a TypeError otherwise - s = str.__str__(s) - else: - if isinstance(s, str) and HAS_UTF8.search(s) is not None: - s = unicode(s, 'utf-8') - elif type(s) not in (str, unicode): - # convert an str subclass instance to exact str - # convert a unicode subclass instance to exact unicode - # raise a TypeError otherwise - if isinstance(s, str): - s = str.__str__(s) - else: - s = unicode.__getnewargs__(s)[0] - def replace(match): - s = match.group(0) - try: - return ESCAPE_DCT[s] - except KeyError: - n = ord(s) - if n < 0x10000: - #return '\\u{0:04x}'.format(n) - return '\\u%04x' % (n,) - else: - # surrogate pair - n -= 0x10000 - s1 = 0xd800 | ((n >> 10) & 0x3ff) - s2 = 0xdc00 | (n & 0x3ff) - #return '\\u{0:04x}\\u{1:04x}'.format(s1, s2) - return '\\u%04x\\u%04x' % (s1, s2) - return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"' - - -encode_basestring_ascii = ( - c_encode_basestring_ascii or py_encode_basestring_ascii) - -class JSONEncoder(object): - """Extensible JSON <http://json.org> encoder for Python data structures. - - Supports the following objects and types by default: - - +-------------------+---------------+ - | Python | JSON | - +===================+===============+ - | dict, namedtuple | object | - +-------------------+---------------+ - | list, tuple | array | - +-------------------+---------------+ - | str, unicode | string | - +-------------------+---------------+ - | int, long, float | number | - +-------------------+---------------+ - | True | true | - +-------------------+---------------+ - | False | false | - +-------------------+---------------+ - | None | null | - +-------------------+---------------+ - - To extend this to recognize other objects, subclass and implement a - ``.default()`` method with another method that returns a serializable - object for ``o`` if possible, otherwise it should call the superclass - implementation (to raise ``TypeError``). - - """ - item_separator = ', ' - key_separator = ': ' - - def __init__(self, skipkeys=False, ensure_ascii=True, - check_circular=True, allow_nan=False, sort_keys=False, - indent=None, separators=None, encoding='utf-8', default=None, - use_decimal=True, namedtuple_as_object=True, - tuple_as_array=True, bigint_as_string=False, - item_sort_key=None, for_json=False, ignore_nan=False, - int_as_string_bitcount=None, iterable_as_array=False): - """Constructor for JSONEncoder, with sensible defaults. - - If skipkeys is false, then it is a TypeError to attempt - encoding of keys that are not str, int, long, float or None. If - skipkeys is True, such items are simply skipped. - - If ensure_ascii is true, the output is guaranteed to be str - objects with all incoming unicode characters escaped. If - ensure_ascii is false, the output will be unicode object. - - If check_circular is true, then lists, dicts, and custom encoded - objects will be checked for circular references during encoding to - prevent an infinite recursion (which would cause an OverflowError). - Otherwise, no such check takes place. - - If allow_nan is true (default: False), then out of range float - values (nan, inf, -inf) will be serialized to - their JavaScript equivalents (NaN, Infinity, -Infinity) - instead of raising a ValueError. See - ignore_nan for ECMA-262 compliant behavior. - - If sort_keys is true, then the output of dictionaries will be - sorted by key; this is useful for regression tests to ensure - that JSON serializations can be compared on a day-to-day basis. - - If indent is a string, then JSON array elements and object members - will be pretty-printed with a newline followed by that string repeated - for each level of nesting. ``None`` (the default) selects the most compact - representation without any newlines. For backwards compatibility with - versions of simplejson earlier than 2.1.0, an integer is also accepted - and is converted to a string with that many spaces. - - If specified, separators should be an (item_separator, key_separator) - tuple. The default is (', ', ': ') if *indent* is ``None`` and - (',', ': ') otherwise. To get the most compact JSON representation, - you should specify (',', ':') to eliminate whitespace. - - If specified, default is a function that gets called for objects - that can't otherwise be serialized. It should return a JSON encodable - version of the object or raise a ``TypeError``. - - If encoding is not None, then all input strings will be - transformed into unicode using that encoding prior to JSON-encoding. - The default is UTF-8. - - If use_decimal is true (default: ``True``), ``decimal.Decimal`` will - be supported directly by the encoder. For the inverse, decode JSON - with ``parse_float=decimal.Decimal``. - - If namedtuple_as_object is true (the default), objects with - ``_asdict()`` methods will be encoded as JSON objects. - - If tuple_as_array is true (the default), tuple (and subclasses) will - be encoded as JSON arrays. - - If *iterable_as_array* is true (default: ``False``), - any object not in the above table that implements ``__iter__()`` - will be encoded as a JSON array. - - If bigint_as_string is true (not the default), ints 2**53 and higher - or lower than -2**53 will be encoded as strings. This is to avoid the - rounding that happens in Javascript otherwise. - - If int_as_string_bitcount is a positive number (n), then int of size - greater than or equal to 2**n or lower than or equal to -2**n will be - encoded as strings. - - If specified, item_sort_key is a callable used to sort the items in - each dictionary. This is useful if you want to sort items other than - in alphabetical order by key. - - If for_json is true (not the default), objects with a ``for_json()`` - method will use the return value of that method for encoding as JSON - instead of the object. - - If *ignore_nan* is true (default: ``False``), then out of range - :class:`float` values (``nan``, ``inf``, ``-inf``) will be serialized - as ``null`` in compliance with the ECMA-262 specification. If true, - this will override *allow_nan*. - - """ - - self.skipkeys = skipkeys - self.ensure_ascii = ensure_ascii - self.check_circular = check_circular - self.allow_nan = allow_nan - self.sort_keys = sort_keys - self.use_decimal = use_decimal - self.namedtuple_as_object = namedtuple_as_object - self.tuple_as_array = tuple_as_array - self.iterable_as_array = iterable_as_array - self.bigint_as_string = bigint_as_string - self.item_sort_key = item_sort_key - self.for_json = for_json - self.ignore_nan = ignore_nan - self.int_as_string_bitcount = int_as_string_bitcount - if indent is not None and not isinstance(indent, string_types): - indent = indent * ' ' - self.indent = indent - if separators is not None: - self.item_separator, self.key_separator = separators - elif indent is not None: - self.item_separator = ',' - if default is not None: - self.default = default - self.encoding = encoding - - def default(self, o): - """Implement this method in a subclass such that it returns - a serializable object for ``o``, or calls the base implementation - (to raise a ``TypeError``). - - For example, to support arbitrary iterators, you could - implement default like this:: - - def default(self, o): - try: - iterable = iter(o) - except TypeError: - pass - else: - return list(iterable) - return JSONEncoder.default(self, o) - - """ - raise TypeError('Object of type %s is not JSON serializable' % - o.__class__.__name__) - - def encode(self, o): - """Return a JSON string representation of a Python data structure. - - >>> from simplejson import JSONEncoder - >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) - '{"foo": ["bar", "baz"]}' - - """ - # This is for extremely simple cases and benchmarks. - if isinstance(o, binary_type): - _encoding = self.encoding - if (_encoding is not None and not (_encoding == 'utf-8')): - o = text_type(o, _encoding) - if isinstance(o, string_types): - if self.ensure_ascii: - return encode_basestring_ascii(o) - else: - return encode_basestring(o) - # This doesn't pass the iterator directly to ''.join() because the - # exceptions aren't as detailed. The list call should be roughly - # equivalent to the PySequence_Fast that ''.join() would do. - chunks = self.iterencode(o) - if not isinstance(chunks, (list, tuple)): - chunks = list(chunks) - if self.ensure_ascii: - return ''.join(chunks) - else: - return u''.join(chunks) - - def iterencode(self, o): - """Encode the given object and yield each string - representation as available. - - For example:: - - for chunk in JSONEncoder().iterencode(bigobject): - mysocket.write(chunk) - - """ - if self.check_circular: - markers = {} - else: - markers = None - if self.ensure_ascii: - _encoder = encode_basestring_ascii - else: - _encoder = encode_basestring - if self.encoding != 'utf-8' and self.encoding is not None: - def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding): - if isinstance(o, binary_type): - o = text_type(o, _encoding) - return _orig_encoder(o) - - def floatstr(o, allow_nan=self.allow_nan, ignore_nan=self.ignore_nan, - _repr=FLOAT_REPR, _inf=PosInf, _neginf=-PosInf): - # Check for specials. Note that this type of test is processor - # and/or platform-specific, so do tests which don't depend on - # the internals. - - if o != o: - text = 'NaN' - elif o == _inf: - text = 'Infinity' - elif o == _neginf: - text = '-Infinity' - else: - if type(o) != float: - # See #118, do not trust custom str/repr - o = float(o) - return _repr(o) - - if ignore_nan: - text = 'null' - elif not allow_nan: - raise ValueError( - "Out of range float values are not JSON compliant: " + - repr(o)) - - return text - - key_memo = {} - int_as_string_bitcount = ( - 53 if self.bigint_as_string else self.int_as_string_bitcount) - if (c_make_encoder is not None and self.indent is None): - _iterencode = c_make_encoder( - markers, self.default, _encoder, self.indent, - self.key_separator, self.item_separator, self.sort_keys, - self.skipkeys, self.allow_nan, key_memo, self.use_decimal, - self.namedtuple_as_object, self.tuple_as_array, - int_as_string_bitcount, - self.item_sort_key, self.encoding, self.for_json, - self.ignore_nan, decimal.Decimal, self.iterable_as_array) - else: - _iterencode = _make_iterencode( - markers, self.default, _encoder, self.indent, floatstr, - self.key_separator, self.item_separator, self.sort_keys, - self.skipkeys, self.use_decimal, - self.namedtuple_as_object, self.tuple_as_array, - int_as_string_bitcount, - self.item_sort_key, self.encoding, self.for_json, - self.iterable_as_array, Decimal=decimal.Decimal) - try: - return _iterencode(o, 0) - finally: - key_memo.clear() - - -class JSONEncoderForHTML(JSONEncoder): - """An encoder that produces JSON safe to embed in HTML. - - To embed JSON content in, say, a script tag on a web page, the - characters &, < and > should be escaped. They cannot be escaped - with the usual entities (e.g. &) because they are not expanded - within <script> tags. - - This class also escapes the line separator and paragraph separator - characters U+2028 and U+2029, irrespective of the ensure_ascii setting, - as these characters are not valid in JavaScript strings (see - http://timelessrepo.com/json-isnt-a-javascript-subset). - """ - - def encode(self, o): - # Override JSONEncoder.encode because it has hacks for - # performance that make things more complicated. - chunks = self.iterencode(o) - if self.ensure_ascii: - return ''.join(chunks) - else: - return u''.join(chunks) - - def iterencode(self, o): - chunks = super(JSONEncoderForHTML, self).iterencode(o) - for chunk in chunks: - chunk = chunk.replace('&', '\\u0026') - chunk = chunk.replace('<', '\\u003c') - chunk = chunk.replace('>', '\\u003e') - - if not self.ensure_ascii: - chunk = chunk.replace(u'\u2028', '\\u2028') - chunk = chunk.replace(u'\u2029', '\\u2029') - - yield chunk - - -def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, - _key_separator, _item_separator, _sort_keys, _skipkeys, - _use_decimal, _namedtuple_as_object, _tuple_as_array, - _int_as_string_bitcount, _item_sort_key, - _encoding,_for_json, - _iterable_as_array, - ## HACK: hand-optimized bytecode; turn globals into locals - _PY3=PY3, - ValueError=ValueError, - string_types=string_types, - Decimal=None, - dict=dict, - float=float, - id=id, - integer_types=integer_types, - isinstance=isinstance, - list=list, - str=str, - tuple=tuple, - iter=iter, - ): - if _use_decimal and Decimal is None: - Decimal = decimal.Decimal - if _item_sort_key and not callable(_item_sort_key): - raise TypeError("item_sort_key must be None or callable") - elif _sort_keys and not _item_sort_key: - _item_sort_key = itemgetter(0) - - if (_int_as_string_bitcount is not None and - (_int_as_string_bitcount <= 0 or - not isinstance(_int_as_string_bitcount, integer_types))): - raise TypeError("int_as_string_bitcount must be a positive integer") - - def call_method(obj, method_name): - method = getattr(obj, method_name, None) - if callable(method): - try: - return (method(),) - except TypeError: - pass - return None - - def _encode_int(value): - skip_quoting = ( - _int_as_string_bitcount is None - or - _int_as_string_bitcount < 1 - ) - if type(value) not in integer_types: - # See #118, do not trust custom str/repr - value = int(value) - if ( - skip_quoting or - (-1 << _int_as_string_bitcount) - < value < - (1 << _int_as_string_bitcount) - ): - return str(value) - return '"' + str(value) + '"' - - def _iterencode_list(lst, _current_indent_level): - if not lst: - yield '[]' - return - if markers is not None: - markerid = id(lst) - if markerid in markers: - raise ValueError("Circular reference detected") - markers[markerid] = lst - buf = '[' - if _indent is not None: - _current_indent_level += 1 - newline_indent = '\n' + (_indent * _current_indent_level) - separator = _item_separator + newline_indent - buf += newline_indent - else: - newline_indent = None - separator = _item_separator - first = True - for value in lst: - if first: - first = False - else: - buf = separator - if isinstance(value, string_types): - yield buf + _encoder(value) - elif _PY3 and isinstance(value, bytes) and _encoding is not None: - yield buf + _encoder(value) - elif isinstance(value, RawJSON): - yield buf + value.encoded_json - elif value is None: - yield buf + 'null' - elif value is True: - yield buf + 'true' - elif value is False: - yield buf + 'false' - elif isinstance(value, integer_types): - yield buf + _encode_int(value) - elif isinstance(value, float): - yield buf + _floatstr(value) - elif _use_decimal and isinstance(value, Decimal): - yield buf + str(value) - else: - yield buf - for_json = _for_json and call_method(value, 'for_json') - if for_json: - chunks = _iterencode(for_json[0], _current_indent_level) - elif isinstance(value, list): - chunks = _iterencode_list(value, _current_indent_level) - else: - _asdict = _namedtuple_as_object and call_method(value, '_asdict') - if _asdict: - dct = _asdict[0] - if not isinstance(dct, dict): - raise TypeError("_asdict() must return a dict, not %s" % (type(dct).__name__,)) - chunks = _iterencode_dict(dct, - _current_indent_level) - elif _tuple_as_array and isinstance(value, tuple): - chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): - chunks = _iterencode_dict(value, _current_indent_level) - else: - chunks = _iterencode(value, _current_indent_level) - for chunk in chunks: - yield chunk - if first: - # iterable_as_array misses the fast path at the top - yield '[]' - else: - if newline_indent is not None: - _current_indent_level -= 1 - yield '\n' + (_indent * _current_indent_level) - yield ']' - if markers is not None: - del markers[markerid] - - def _stringify_key(key): - if isinstance(key, string_types): # pragma: no cover - pass - elif _PY3 and isinstance(key, bytes) and _encoding is not None: - key = str(key, _encoding) - elif isinstance(key, float): - key = _floatstr(key) - elif key is True: - key = 'true' - elif key is False: - key = 'false' - elif key is None: - key = 'null' - elif isinstance(key, integer_types): - if type(key) not in integer_types: - # See #118, do not trust custom str/repr - key = int(key) - key = str(key) - elif _use_decimal and isinstance(key, Decimal): - key = str(key) - elif _skipkeys: - key = None - else: - raise TypeError('keys must be str, int, float, bool or None, ' - 'not %s' % key.__class__.__name__) - return key - - def _iterencode_dict(dct, _current_indent_level): - if not dct: - yield '{}' - return - if markers is not None: - markerid = id(dct) - if markerid in markers: - raise ValueError("Circular reference detected") - markers[markerid] = dct - yield '{' - if _indent is not None: - _current_indent_level += 1 - newline_indent = '\n' + (_indent * _current_indent_level) - item_separator = _item_separator + newline_indent - yield newline_indent - else: - newline_indent = None - item_separator = _item_separator - first = True - if _PY3: - iteritems = dct.items() - else: - iteritems = dct.iteritems() - if _item_sort_key: - items = [] - for k, v in dct.items(): - if not isinstance(k, string_types): - k = _stringify_key(k) - if k is None: - continue - items.append((k, v)) - items.sort(key=_item_sort_key) - else: - items = iteritems - for key, value in items: - if not (_item_sort_key or isinstance(key, string_types)): - key = _stringify_key(key) - if key is None: - # _skipkeys must be True - continue - if first: - first = False - else: - yield item_separator - yield _encoder(key) - yield _key_separator - if isinstance(value, string_types): - yield _encoder(value) - elif _PY3 and isinstance(value, bytes) and _encoding is not None: - yield _encoder(value) - elif isinstance(value, RawJSON): - yield value.encoded_json - elif value is None: - yield 'null' - elif value is True: - yield 'true' - elif value is False: - yield 'false' - elif isinstance(value, integer_types): - yield _encode_int(value) - elif isinstance(value, float): - yield _floatstr(value) - elif _use_decimal and isinstance(value, Decimal): - yield str(value) - else: - for_json = _for_json and call_method(value, 'for_json') - if for_json: - chunks = _iterencode(for_json[0], _current_indent_level) - elif isinstance(value, list): - chunks = _iterencode_list(value, _current_indent_level) - else: - _asdict = _namedtuple_as_object and call_method(value, '_asdict') - if _asdict: - dct = _asdict[0] - if not isinstance(dct, dict): - raise TypeError("_asdict() must return a dict, not %s" % (type(dct).__name__,)) - chunks = _iterencode_dict(dct, - _current_indent_level) - elif _tuple_as_array and isinstance(value, tuple): - chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): - chunks = _iterencode_dict(value, _current_indent_level) - else: - chunks = _iterencode(value, _current_indent_level) - for chunk in chunks: - yield chunk - if newline_indent is not None: - _current_indent_level -= 1 - yield '\n' + (_indent * _current_indent_level) - yield '}' - if markers is not None: - del markers[markerid] - - def _iterencode(o, _current_indent_level): - if isinstance(o, string_types): - yield _encoder(o) - elif _PY3 and isinstance(o, bytes) and _encoding is not None: - yield _encoder(o) - elif isinstance(o, RawJSON): - yield o.encoded_json - elif o is None: - yield 'null' - elif o is True: - yield 'true' - elif o is False: - yield 'false' - elif isinstance(o, integer_types): - yield _encode_int(o) - elif isinstance(o, float): - yield _floatstr(o) - else: - for_json = _for_json and call_method(o, 'for_json') - if for_json: - for chunk in _iterencode(for_json[0], _current_indent_level): - yield chunk - elif isinstance(o, list): - for chunk in _iterencode_list(o, _current_indent_level): - yield chunk - else: - _asdict = _namedtuple_as_object and call_method(o, '_asdict') - if _asdict: - dct = _asdict[0] - if not isinstance(dct, dict): - raise TypeError("_asdict() must return a dict, not %s" % (type(dct).__name__,)) - for chunk in _iterencode_dict(dct, _current_indent_level): - yield chunk - elif (_tuple_as_array and isinstance(o, tuple)): - for chunk in _iterencode_list(o, _current_indent_level): - yield chunk - elif isinstance(o, dict): - for chunk in _iterencode_dict(o, _current_indent_level): - yield chunk - elif _use_decimal and isinstance(o, Decimal): - yield str(o) - else: - while _iterable_as_array: - # Markers are not checked here because it is valid for - # an iterable to return self. - try: - o = iter(o) - except TypeError: - break - for chunk in _iterencode_list(o, _current_indent_level): - yield chunk - return - if markers is not None: - markerid = id(o) - if markerid in markers: - raise ValueError("Circular reference detected") - markers[markerid] = o - o = _default(o) - for chunk in _iterencode(o, _current_indent_level): - yield chunk - if markers is not None: - del markers[markerid] - - return _iterencode diff --git a/contrib/python/simplejson/py3/simplejson/errors.py b/contrib/python/simplejson/py3/simplejson/errors.py deleted file mode 100644 index b97ab1e913f..00000000000 --- a/contrib/python/simplejson/py3/simplejson/errors.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Error classes used by simplejson -""" -__all__ = ['JSONDecodeError'] - - -def linecol(doc, pos): - lineno = doc.count('\n', 0, pos) + 1 - if lineno == 1: - colno = pos + 1 - else: - colno = pos - doc.rindex('\n', 0, pos) - return lineno, colno - - -def errmsg(msg, doc, pos, end=None): - lineno, colno = linecol(doc, pos) - msg = msg.replace('%r', repr(doc[pos:pos + 1])) - if end is None: - fmt = '%s: line %d column %d (char %d)' - return fmt % (msg, lineno, colno, pos) - endlineno, endcolno = linecol(doc, end) - fmt = '%s: line %d column %d - line %d column %d (char %d - %d)' - return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end) - - -class JSONDecodeError(ValueError): - """Subclass of ValueError with the following additional properties: - - msg: The unformatted error message - doc: The JSON document being parsed - pos: The start index of doc where parsing failed - end: The end index of doc where parsing failed (may be None) - lineno: The line corresponding to pos - colno: The column corresponding to pos - endlineno: The line corresponding to end (may be None) - endcolno: The column corresponding to end (may be None) - - """ - # Note that this exception is used from _speedups - def __init__(self, msg, doc, pos, end=None): - ValueError.__init__(self, errmsg(msg, doc, pos, end=end)) - self.msg = msg - self.doc = doc - self.pos = pos - self.end = end - self.lineno, self.colno = linecol(doc, pos) - if end is not None: - self.endlineno, self.endcolno = linecol(doc, end) - else: - self.endlineno, self.endcolno = None, None - - def __reduce__(self): - return self.__class__, (self.msg, self.doc, self.pos, self.end) diff --git a/contrib/python/simplejson/py3/simplejson/ordered_dict.py b/contrib/python/simplejson/py3/simplejson/ordered_dict.py deleted file mode 100644 index d5a55ebd0f7..00000000000 --- a/contrib/python/simplejson/py3/simplejson/ordered_dict.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Drop-in replacement for collections.OrderedDict by Raymond Hettinger - -http://code.activestate.com/recipes/576693/ - -""" -from UserDict import DictMixin - -class OrderedDict(dict, DictMixin): - - def __init__(self, *args, **kwds): - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__end - except AttributeError: - self.clear() - self.update(*args, **kwds) - - def clear(self): - self.__end = end = [] - end += [None, end, end] # sentinel node for doubly linked list - self.__map = {} # key --> [key, prev, next] - dict.clear(self) - - def __setitem__(self, key, value): - if key not in self: - end = self.__end - curr = end[1] - curr[2] = end[1] = self.__map[key] = [key, curr, end] - dict.__setitem__(self, key, value) - - def __delitem__(self, key): - dict.__delitem__(self, key) - key, prev, next = self.__map.pop(key) - prev[2] = next - next[1] = prev - - def __iter__(self): - end = self.__end - curr = end[2] - while curr is not end: - yield curr[0] - curr = curr[2] - - def __reversed__(self): - end = self.__end - curr = end[1] - while curr is not end: - yield curr[0] - curr = curr[1] - - def popitem(self, last=True): - if not self: - raise KeyError('dictionary is empty') - key = reversed(self).next() if last else iter(self).next() - value = self.pop(key) - return key, value - - def __reduce__(self): - items = [[k, self[k]] for k in self] - tmp = self.__map, self.__end - del self.__map, self.__end - inst_dict = vars(self).copy() - self.__map, self.__end = tmp - if inst_dict: - return (self.__class__, (items,), inst_dict) - return self.__class__, (items,) - - def keys(self): - return list(self) - - setdefault = DictMixin.setdefault - update = DictMixin.update - pop = DictMixin.pop - values = DictMixin.values - items = DictMixin.items - iterkeys = DictMixin.iterkeys - itervalues = DictMixin.itervalues - iteritems = DictMixin.iteritems - - def __repr__(self): - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, self.items()) - - def copy(self): - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - d = cls() - for key in iterable: - d[key] = value - return d - - def __eq__(self, other): - if isinstance(other, OrderedDict): - return len(self)==len(other) and \ - all(p==q for p, q in zip(self.items(), other.items())) - return dict.__eq__(self, other) - - def __ne__(self, other): - return not self == other diff --git a/contrib/python/simplejson/py3/simplejson/raw_json.py b/contrib/python/simplejson/py3/simplejson/raw_json.py deleted file mode 100644 index 2071a70206b..00000000000 --- a/contrib/python/simplejson/py3/simplejson/raw_json.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Implementation of RawJSON -""" - -class RawJSON(object): - """Wrap an encoded JSON document for direct embedding in the output - - """ - def __init__(self, encoded_json): - self.encoded_json = encoded_json diff --git a/contrib/python/simplejson/py3/simplejson/scanner.py b/contrib/python/simplejson/py3/simplejson/scanner.py deleted file mode 100644 index 34710d68c6d..00000000000 --- a/contrib/python/simplejson/py3/simplejson/scanner.py +++ /dev/null @@ -1,85 +0,0 @@ -"""JSON token scanner -""" -import re -from .errors import JSONDecodeError -def _import_c_make_scanner(): - try: - from ._speedups import make_scanner - return make_scanner - except ImportError: - return None -c_make_scanner = _import_c_make_scanner() - -__all__ = ['make_scanner', 'JSONDecodeError'] - -NUMBER_RE = re.compile( - r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?', - (re.VERBOSE | re.MULTILINE | re.DOTALL)) - - -def py_make_scanner(context): - parse_object = context.parse_object - parse_array = context.parse_array - parse_string = context.parse_string - match_number = NUMBER_RE.match - encoding = context.encoding - strict = context.strict - parse_float = context.parse_float - parse_int = context.parse_int - parse_constant = context.parse_constant - object_hook = context.object_hook - object_pairs_hook = context.object_pairs_hook - memo = context.memo - - def _scan_once(string, idx): - errmsg = 'Expecting value' - try: - nextchar = string[idx] - except IndexError: - raise JSONDecodeError(errmsg, string, idx) - - if nextchar == '"': - return parse_string(string, idx + 1, encoding, strict) - elif nextchar == '{': - return parse_object((string, idx + 1), encoding, strict, - _scan_once, object_hook, object_pairs_hook, memo) - elif nextchar == '[': - return parse_array((string, idx + 1), _scan_once) - elif nextchar == 'n' and string[idx:idx + 4] == 'null': - return None, idx + 4 - elif nextchar == 't' and string[idx:idx + 4] == 'true': - return True, idx + 4 - elif nextchar == 'f' and string[idx:idx + 5] == 'false': - return False, idx + 5 - - m = match_number(string, idx) - if m is not None: - integer, frac, exp = m.groups() - if frac or exp: - res = parse_float(integer + (frac or '') + (exp or '')) - else: - res = parse_int(integer) - return res, m.end() - elif parse_constant and nextchar == 'N' and string[idx:idx + 3] == 'NaN': - return parse_constant('NaN'), idx + 3 - elif parse_constant and nextchar == 'I' and string[idx:idx + 8] == 'Infinity': - return parse_constant('Infinity'), idx + 8 - elif parse_constant and nextchar == '-' and string[idx:idx + 9] == '-Infinity': - return parse_constant('-Infinity'), idx + 9 - else: - raise JSONDecodeError(errmsg, string, idx) - - def scan_once(string, idx): - if idx < 0: - # Ensure the same behavior as the C speedup, otherwise - # this would work for *some* negative string indices due - # to the behavior of __getitem__ for strings. #98 - raise JSONDecodeError('Expecting value', string, idx) - try: - return _scan_once(string, idx) - finally: - memo.clear() - - return scan_once - -make_scanner = c_make_scanner or py_make_scanner diff --git a/contrib/python/simplejson/py3/simplejson/tests/__init__.py b/contrib/python/simplejson/py3/simplejson/tests/__init__.py deleted file mode 100644 index d9986d033a0..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/__init__.py +++ /dev/null @@ -1,93 +0,0 @@ -from __future__ import absolute_import -import unittest -import sys -import os - - -class NoExtensionTestSuite(unittest.TestSuite): - def run(self, result): - import simplejson - - simplejson._toggle_speedups(False) - result = unittest.TestSuite.run(self, result) - simplejson._toggle_speedups(True) - return result - - -class TestMissingSpeedups(unittest.TestCase): - def runTest(self): - if hasattr(sys, "pypy_translation_info"): - "PyPy doesn't need speedups! :)" - elif getattr(getattr(sys, "implementation", None), "name", None) == "graalpy": - "GraalPy doesn't need speedups! :)" - elif hasattr(self, "skipTest"): - self.skipTest("_speedups.so is missing!") - - -def additional_tests(suite=None, project_dir=None): - import simplejson - import simplejson.encoder - import simplejson.decoder - - if suite is None: - suite = unittest.TestSuite() - try: - import doctest - except ImportError: - if sys.version_info < (2, 7): - # doctests in 2.6 depends on cStringIO - return suite - raise - for mod in (simplejson, simplejson.encoder, simplejson.decoder): - suite.addTest(doctest.DocTestSuite(mod)) - if project_dir is not None: - suite.addTest( - doctest.DocFileSuite( - os.path.join(project_dir, "index.rst"), module_relative=False - ) - ) - return suite - - -def all_tests_suite(project_dir=None): - def get_suite(): - suite_names = [ - "simplejson.tests.%s" % (os.path.splitext(f)[0],) - for f in os.listdir(os.path.dirname(__file__)) - if f.startswith("test_") and f.endswith(".py") - ] - return additional_tests( - suite=unittest.TestLoader().loadTestsFromNames(suite_names), - project_dir=project_dir, - ) - - suite = get_suite() - import simplejson - - if simplejson._import_c_make_encoder() is None: - suite.addTest(TestMissingSpeedups()) - else: - suite = unittest.TestSuite( - [ - suite, - NoExtensionTestSuite([get_suite()]), - ] - ) - return suite - - -def main(project_dir=None): - runner = unittest.TextTestRunner(verbosity=1 + sys.argv.count("-v")) - suite = all_tests_suite(project_dir=project_dir) - raise SystemExit(not runner.run(suite).wasSuccessful()) - - -if __name__ == "__main__": - import os - import sys - - project_dir = os.path.dirname( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - ) - sys.path.insert(0, project_dir) - main(project_dir=project_dir) diff --git a/contrib/python/simplejson/py3/simplejson/tests/_cibw_runner.py b/contrib/python/simplejson/py3/simplejson/tests/_cibw_runner.py deleted file mode 100644 index bffb70b8add..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/_cibw_runner.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Internal module for running tests from cibuildwheel""" - -import sys -import simplejson.tests - -if __name__ == '__main__': - simplejson.tests.main(project_dir=sys.argv[1]) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_bigint_as_string.py b/contrib/python/simplejson/py3/simplejson/tests/test_bigint_as_string.py deleted file mode 100644 index 2cf2cc2966e..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_bigint_as_string.py +++ /dev/null @@ -1,67 +0,0 @@ -from unittest import TestCase - -import simplejson as json - - -class TestBigintAsString(TestCase): - # Python 2.5, at least the one that ships on Mac OS X, calculates - # 2 ** 53 as 0! It manages to calculate 1 << 53 correctly. - values = [(200, 200), - ((1 << 53) - 1, 9007199254740991), - ((1 << 53), '9007199254740992'), - ((1 << 53) + 1, '9007199254740993'), - (-100, -100), - ((-1 << 53), '-9007199254740992'), - ((-1 << 53) - 1, '-9007199254740993'), - ((-1 << 53) + 1, -9007199254740991)] - - options = ( - {"bigint_as_string": True}, - {"int_as_string_bitcount": 53} - ) - - def test_ints(self): - for opts in self.options: - for val, expect in self.values: - self.assertEqual( - val, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, **opts))) - - def test_lists(self): - for opts in self.options: - for val, expect in self.values: - val = [val, val] - expect = [expect, expect] - self.assertEqual( - val, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, **opts))) - - def test_dicts(self): - for opts in self.options: - for val, expect in self.values: - val = {'k': val} - expect = {'k': expect} - self.assertEqual( - val, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, **opts))) - - def test_dict_keys(self): - for opts in self.options: - for val, _ in self.values: - expect = {str(val): 'value'} - val = {val: 'value'} - self.assertEqual( - expect, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, **opts))) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_bitsize_int_as_string.py b/contrib/python/simplejson/py3/simplejson/tests/test_bitsize_int_as_string.py deleted file mode 100644 index fd7d103a6d8..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_bitsize_int_as_string.py +++ /dev/null @@ -1,73 +0,0 @@ -from unittest import TestCase - -import simplejson as json - - -class TestBitSizeIntAsString(TestCase): - # Python 2.5, at least the one that ships on Mac OS X, calculates - # 2 ** 31 as 0! It manages to calculate 1 << 31 correctly. - values = [ - (200, 200), - ((1 << 31) - 1, (1 << 31) - 1), - ((1 << 31), str(1 << 31)), - ((1 << 31) + 1, str((1 << 31) + 1)), - (-100, -100), - ((-1 << 31), str(-1 << 31)), - ((-1 << 31) - 1, str((-1 << 31) - 1)), - ((-1 << 31) + 1, (-1 << 31) + 1), - ] - - def test_invalid_counts(self): - for n in ['foo', -1, 0, 1.0]: - self.assertRaises( - TypeError, - json.dumps, 0, int_as_string_bitcount=n) - - def test_ints_outside_range_fails(self): - self.assertNotEqual( - str(1 << 15), - json.loads(json.dumps(1 << 15, int_as_string_bitcount=16)), - ) - - def test_ints(self): - for val, expect in self.values: - self.assertEqual( - val, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, int_as_string_bitcount=31)), - ) - - def test_lists(self): - for val, expect in self.values: - val = [val, val] - expect = [expect, expect] - self.assertEqual( - val, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, int_as_string_bitcount=31))) - - def test_dicts(self): - for val, expect in self.values: - val = {'k': val} - expect = {'k': expect} - self.assertEqual( - val, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, int_as_string_bitcount=31))) - - def test_dict_keys(self): - for val, _ in self.values: - expect = {str(val): 'value'} - val = {val: 'value'} - self.assertEqual( - expect, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, int_as_string_bitcount=31))) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_check_circular.py b/contrib/python/simplejson/py3/simplejson/tests/test_check_circular.py deleted file mode 100644 index af6463d6d88..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_check_circular.py +++ /dev/null @@ -1,30 +0,0 @@ -from unittest import TestCase -import simplejson as json - -def default_iterable(obj): - return list(obj) - -class TestCheckCircular(TestCase): - def test_circular_dict(self): - dct = {} - dct['a'] = dct - self.assertRaises(ValueError, json.dumps, dct) - - def test_circular_list(self): - lst = [] - lst.append(lst) - self.assertRaises(ValueError, json.dumps, lst) - - def test_circular_composite(self): - dct2 = {} - dct2['a'] = [] - dct2['a'].append(dct2) - self.assertRaises(ValueError, json.dumps, dct2) - - def test_circular_default(self): - json.dumps([set()], default=default_iterable) - self.assertRaises(TypeError, json.dumps, [set()]) - - def test_circular_off_default(self): - json.dumps([set()], default=default_iterable, check_circular=False) - self.assertRaises(TypeError, json.dumps, [set()], check_circular=False) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_decimal.py b/contrib/python/simplejson/py3/simplejson/tests/test_decimal.py deleted file mode 100644 index 2b0940b1527..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_decimal.py +++ /dev/null @@ -1,71 +0,0 @@ -import decimal -from decimal import Decimal -from unittest import TestCase -from simplejson.compat import StringIO, reload_module - -import simplejson as json - -class TestDecimal(TestCase): - NUMS = "1.0", "10.00", "1.1", "1234567890.1234567890", "500" - def dumps(self, obj, **kw): - sio = StringIO() - json.dump(obj, sio, **kw) - res = json.dumps(obj, **kw) - self.assertEqual(res, sio.getvalue()) - return res - - def loads(self, s, **kw): - sio = StringIO(s) - res = json.loads(s, **kw) - self.assertEqual(res, json.load(sio, **kw)) - return res - - def test_decimal_encode(self): - for d in map(Decimal, self.NUMS): - self.assertEqual(self.dumps(d, use_decimal=True), str(d)) - - def test_decimal_decode(self): - for s in self.NUMS: - self.assertEqual(self.loads(s, parse_float=Decimal), Decimal(s)) - - def test_stringify_key(self): - for d in map(Decimal, self.NUMS): - v = {d: d} - self.assertEqual( - self.loads( - self.dumps(v, use_decimal=True), parse_float=Decimal), - {str(d): d}) - - def test_decimal_roundtrip(self): - for d in map(Decimal, self.NUMS): - # The type might not be the same (int and Decimal) but they - # should still compare equal. - for v in [d, [d], {'': d}]: - self.assertEqual( - self.loads( - self.dumps(v, use_decimal=True), parse_float=Decimal), - v) - - def test_decimal_defaults(self): - d = Decimal('1.1') - # use_decimal=True is the default - self.assertRaises(TypeError, json.dumps, d, use_decimal=False) - self.assertEqual('1.1', json.dumps(d)) - self.assertEqual('1.1', json.dumps(d, use_decimal=True)) - self.assertRaises(TypeError, json.dump, d, StringIO(), - use_decimal=False) - sio = StringIO() - json.dump(d, sio) - self.assertEqual('1.1', sio.getvalue()) - sio = StringIO() - json.dump(d, sio, use_decimal=True) - self.assertEqual('1.1', sio.getvalue()) - - def test_decimal_reload(self): - # Simulate a subinterpreter that reloads the Python modules but not - # the C code https://github.com/simplejson/simplejson/issues/34 - global Decimal - Decimal = reload_module(decimal).Decimal - import simplejson.encoder - simplejson.encoder.Decimal = Decimal - self.test_decimal_roundtrip() diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_decode.py b/contrib/python/simplejson/py3/simplejson/tests/test_decode.py deleted file mode 100644 index 317b4f98d55..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_decode.py +++ /dev/null @@ -1,127 +0,0 @@ -from __future__ import absolute_import -import decimal -from unittest import TestCase - -import sys -import simplejson as json -from simplejson.compat import StringIO, b, binary_type -from simplejson import OrderedDict - -class MisbehavingBytesSubtype(binary_type): - def decode(self, encoding=None): - return "bad decode" - def __str__(self): - return "bad __str__" - def __bytes__(self): - return b("bad __bytes__") - -class TestDecode(TestCase): - if not hasattr(TestCase, 'assertIs'): - def assertIs(self, a, b): - self.assertTrue(a is b, '%r is %r' % (a, b)) - - def test_decimal(self): - rval = json.loads('1.1', parse_float=decimal.Decimal) - self.assertTrue(isinstance(rval, decimal.Decimal)) - self.assertEqual(rval, decimal.Decimal('1.1')) - - def test_float(self): - rval = json.loads('1', parse_int=float) - self.assertTrue(isinstance(rval, float)) - self.assertEqual(rval, 1.0) - - def test_decoder_optimizations(self): - # Several optimizations were made that skip over calls to - # the whitespace regex, so this test is designed to try and - # exercise the uncommon cases. The array cases are already covered. - rval = json.loads('{ "key" : "value" , "k":"v" }') - self.assertEqual(rval, {"key":"value", "k":"v"}) - - def test_empty_objects(self): - s = '{}' - self.assertEqual(json.loads(s), eval(s)) - s = '[]' - self.assertEqual(json.loads(s), eval(s)) - s = '""' - self.assertEqual(json.loads(s), eval(s)) - - def test_object_pairs_hook(self): - s = '{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}' - p = [("xkd", 1), ("kcw", 2), ("art", 3), ("hxm", 4), - ("qrt", 5), ("pad", 6), ("hoy", 7)] - self.assertEqual(json.loads(s), eval(s)) - self.assertEqual(json.loads(s, object_pairs_hook=lambda x: x), p) - self.assertEqual(json.load(StringIO(s), - object_pairs_hook=lambda x: x), p) - od = json.loads(s, object_pairs_hook=OrderedDict) - self.assertEqual(od, OrderedDict(p)) - self.assertEqual(type(od), OrderedDict) - # the object_pairs_hook takes priority over the object_hook - self.assertEqual(json.loads(s, - object_pairs_hook=OrderedDict, - object_hook=lambda x: None), - OrderedDict(p)) - - def check_keys_reuse(self, source, loads): - rval = loads(source) - (a, b), (c, d) = sorted(rval[0]), sorted(rval[1]) - self.assertIs(a, c) - self.assertIs(b, d) - - def test_keys_reuse_str(self): - s = u'[{"a_key": 1, "b_\xe9": 2}, {"a_key": 3, "b_\xe9": 4}]'.encode('utf8') - self.check_keys_reuse(s, json.loads) - - def test_keys_reuse_unicode(self): - s = u'[{"a_key": 1, "b_\xe9": 2}, {"a_key": 3, "b_\xe9": 4}]' - self.check_keys_reuse(s, json.loads) - - def test_empty_strings(self): - self.assertEqual(json.loads('""'), "") - self.assertEqual(json.loads(u'""'), u"") - self.assertEqual(json.loads('[""]'), [""]) - self.assertEqual(json.loads(u'[""]'), [u""]) - - def test_raw_decode(self): - cls = json.decoder.JSONDecoder - self.assertEqual( - ({'a': {}}, 9), - cls().raw_decode("{\"a\": {}}")) - # http://code.google.com/p/simplejson/issues/detail?id=85 - self.assertEqual( - ({'a': {}}, 9), - cls(object_pairs_hook=dict).raw_decode("{\"a\": {}}")) - # https://github.com/simplejson/simplejson/pull/38 - self.assertEqual( - ({'a': {}}, 11), - cls().raw_decode(" \n{\"a\": {}}")) - - def test_bytes_decode(self): - cls = json.decoder.JSONDecoder - data = b('"\xe2\x82\xac"') - self.assertEqual(cls().decode(data), u'\u20ac') - self.assertEqual(cls(encoding='latin1').decode(data), u'\xe2\x82\xac') - self.assertEqual(cls(encoding=None).decode(data), u'\u20ac') - - data = MisbehavingBytesSubtype(b('"\xe2\x82\xac"')) - self.assertEqual(cls().decode(data), u'\u20ac') - self.assertEqual(cls(encoding='latin1').decode(data), u'\xe2\x82\xac') - self.assertEqual(cls(encoding=None).decode(data), u'\u20ac') - - def test_bounds_checking(self): - # https://github.com/simplejson/simplejson/issues/98 - j = json.decoder.JSONDecoder() - for i in [4, 5, 6, -1, -2, -3, -4, -5, -6]: - self.assertRaises(ValueError, j.scan_once, '1234', i) - self.assertRaises(ValueError, j.raw_decode, '1234', i) - x, y = sorted(['128931233', '472389423'], key=id) - diff = id(x) - id(y) - self.assertRaises(ValueError, j.scan_once, y, diff) - self.assertRaises(ValueError, j.raw_decode, y, i) - - def test_bounded_int(self): - # SJ-PT-23-03, limit quadratic number parsing per Python 3.11 - max_str_digits = getattr(sys, 'get_int_max_str_digits', lambda: 4300)() - s = '1' + '0' * (max_str_digits - 1) - self.assertEqual(json.loads(s), int(s)) - self.assertRaises(ValueError, json.loads, s + '0') diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_default.py b/contrib/python/simplejson/py3/simplejson/tests/test_default.py deleted file mode 100644 index d1eacb8c4c3..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_default.py +++ /dev/null @@ -1,9 +0,0 @@ -from unittest import TestCase - -import simplejson as json - -class TestDefault(TestCase): - def test_default(self): - self.assertEqual( - json.dumps(type, default=repr), - json.dumps(repr(type))) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_dump.py b/contrib/python/simplejson/py3/simplejson/tests/test_dump.py deleted file mode 100644 index 5628489f268..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_dump.py +++ /dev/null @@ -1,255 +0,0 @@ -from unittest import TestCase -from simplejson.compat import StringIO, long_type, b, binary_type, text_type, PY3 -import simplejson as json - -class MisbehavingTextSubtype(text_type): - def __str__(self): - return "FAIL!" - -class MisbehavingBytesSubtype(binary_type): - def decode(self, encoding=None): - return "bad decode" - def __str__(self): - return "bad __str__" - def __bytes__(self): - return b("bad __bytes__") - -def as_text_type(s): - if PY3 and isinstance(s, bytes): - return s.decode('ascii') - return s - -def decode_iso_8859_15(b): - return b.decode('iso-8859-15') - -class TestDump(TestCase): - def test_dump(self): - sio = StringIO() - json.dump({}, sio) - self.assertEqual(sio.getvalue(), '{}') - - def test_constants(self): - for c in [None, True, False]: - self.assertTrue(json.loads(json.dumps(c)) is c) - self.assertTrue(json.loads(json.dumps([c]))[0] is c) - self.assertTrue(json.loads(json.dumps({'a': c}))['a'] is c) - - def test_stringify_key(self): - items = [(b('bytes'), 'bytes'), - (1.0, '1.0'), - (10, '10'), - (True, 'true'), - (False, 'false'), - (None, 'null'), - (long_type(100), '100')] - for k, expect in items: - self.assertEqual( - json.loads(json.dumps({k: expect})), - {expect: expect}) - self.assertEqual( - json.loads(json.dumps({k: expect}, sort_keys=True)), - {expect: expect}) - self.assertRaises(TypeError, json.dumps, {json: 1}) - for v in [{}, {'other': 1}, {b('derp'): 1, 'herp': 2}]: - for sort_keys in [False, True]: - v0 = dict(v) - v0[json] = 1 - v1 = dict((as_text_type(key), val) for (key, val) in v.items()) - self.assertEqual( - json.loads(json.dumps(v0, skipkeys=True, sort_keys=sort_keys)), - v1) - self.assertEqual( - json.loads(json.dumps({'': v0}, skipkeys=True, sort_keys=sort_keys)), - {'': v1}) - self.assertEqual( - json.loads(json.dumps([v0], skipkeys=True, sort_keys=sort_keys)), - [v1]) - - def test_dumps(self): - self.assertEqual(json.dumps({}), '{}') - - def test_encode_truefalse(self): - self.assertEqual(json.dumps( - {True: False, False: True}, sort_keys=True), - '{"false": true, "true": false}') - self.assertEqual( - # load first because the keys are not sorted - json.loads(json.dumps({'k1': {False: 5}, 'k2': {0: 5}})), - {'k1': {'false': 5}, 'k2': {'0': 5}}, - ) - - self.assertEqual( - json.dumps( - {2: 3.0, - 4.0: long_type(5), - False: 1, - long_type(6): True, - "7": 0}, - sort_keys=True), - '{"2": 3.0, "4.0": 5, "6": true, "7": 0, "false": 1}') - - def test_ordered_dict(self): - # http://bugs.python.org/issue6105 - items = [('one', 1), ('two', 2), ('three', 3), ('four', 4), ('five', 5)] - s = json.dumps(json.OrderedDict(items)) - self.assertEqual( - s, - '{"one": 1, "two": 2, "three": 3, "four": 4, "five": 5}') - - def test_indent_unknown_type_acceptance(self): - """ - A test against the regression mentioned at `github issue 29`_. - - The indent parameter should accept any type which pretends to be - an instance of int or long when it comes to being multiplied by - strings, even if it is not actually an int or long, for - backwards compatibility. - - .. _github issue 29: - http://github.com/simplejson/simplejson/issue/29 - """ - - class AwesomeInt(object): - """An awesome reimplementation of integers""" - - def __init__(self, *args, **kwargs): - if len(args) > 0: - # [construct from literals, objects, etc.] - # ... - - # Finally, if args[0] is an integer, store it - if isinstance(args[0], int): - self._int = args[0] - - # [various methods] - - def __mul__(self, other): - # [various ways to multiply AwesomeInt objects] - # ... finally, if the right-hand operand is not awesome enough, - # try to do a normal integer multiplication - if hasattr(self, '_int'): - return self._int * other - else: - raise NotImplementedError("To do non-awesome things with" - " this object, please construct it from an integer!") - - s = json.dumps([0, 1, 2], indent=AwesomeInt(3)) - self.assertEqual(s, '[\n 0,\n 1,\n 2\n]') - - def test_accumulator(self): - # the C API uses an accumulator that collects after 100,000 appends - lst = [0] * 100000 - self.assertEqual(json.loads(json.dumps(lst)), lst) - - def test_sort_keys(self): - # https://github.com/simplejson/simplejson/issues/106 - for num_keys in range(2, 32): - p = dict((str(x), x) for x in range(num_keys)) - sio = StringIO() - json.dump(p, sio, sort_keys=True) - self.assertEqual(sio.getvalue(), json.dumps(p, sort_keys=True)) - self.assertEqual(json.loads(sio.getvalue()), p) - - def test_misbehaving_text_subtype(self): - # https://github.com/simplejson/simplejson/issues/185 - text = "this is some text" - self.assertEqual( - json.dumps(MisbehavingTextSubtype(text)), - json.dumps(text) - ) - self.assertEqual( - json.dumps([MisbehavingTextSubtype(text)]), - json.dumps([text]) - ) - self.assertEqual( - json.dumps({MisbehavingTextSubtype(text): 42}), - json.dumps({text: 42}) - ) - - def test_misbehaving_bytes_subtype(self): - data = b("this is some data \xe2\x82\xac") - self.assertEqual( - json.dumps(MisbehavingBytesSubtype(data)), - json.dumps(data) - ) - self.assertEqual( - json.dumps([MisbehavingBytesSubtype(data)]), - json.dumps([data]) - ) - self.assertEqual( - json.dumps({MisbehavingBytesSubtype(data): 42}), - json.dumps({data: 42}) - ) - - def test_bytes_toplevel(self): - self.assertEqual(json.dumps(b('\xe2\x82\xac')), r'"\u20ac"') - self.assertRaises(UnicodeDecodeError, json.dumps, b('\xa4')) - self.assertEqual(json.dumps(b('\xa4'), encoding='iso-8859-1'), - r'"\u00a4"') - self.assertEqual(json.dumps(b('\xa4'), encoding='iso-8859-15'), - r'"\u20ac"') - if PY3: - self.assertRaises(TypeError, json.dumps, b('\xe2\x82\xac'), - encoding=None) - self.assertRaises(TypeError, json.dumps, b('\xa4'), - encoding=None) - self.assertEqual(json.dumps(b('\xa4'), encoding=None, - default=decode_iso_8859_15), - r'"\u20ac"') - else: - self.assertEqual(json.dumps(b('\xe2\x82\xac'), encoding=None), - r'"\u20ac"') - self.assertRaises(UnicodeDecodeError, json.dumps, b('\xa4'), - encoding=None) - self.assertRaises(UnicodeDecodeError, json.dumps, b('\xa4'), - encoding=None, default=decode_iso_8859_15) - - def test_bytes_nested(self): - self.assertEqual(json.dumps([b('\xe2\x82\xac')]), r'["\u20ac"]') - self.assertRaises(UnicodeDecodeError, json.dumps, [b('\xa4')]) - self.assertEqual(json.dumps([b('\xa4')], encoding='iso-8859-1'), - r'["\u00a4"]') - self.assertEqual(json.dumps([b('\xa4')], encoding='iso-8859-15'), - r'["\u20ac"]') - if PY3: - self.assertRaises(TypeError, json.dumps, [b('\xe2\x82\xac')], - encoding=None) - self.assertRaises(TypeError, json.dumps, [b('\xa4')], - encoding=None) - self.assertEqual(json.dumps([b('\xa4')], encoding=None, - default=decode_iso_8859_15), - r'["\u20ac"]') - else: - self.assertEqual(json.dumps([b('\xe2\x82\xac')], encoding=None), - r'["\u20ac"]') - self.assertRaises(UnicodeDecodeError, json.dumps, [b('\xa4')], - encoding=None) - self.assertRaises(UnicodeDecodeError, json.dumps, [b('\xa4')], - encoding=None, default=decode_iso_8859_15) - - def test_bytes_key(self): - self.assertEqual(json.dumps({b('\xe2\x82\xac'): 42}), r'{"\u20ac": 42}') - self.assertRaises(UnicodeDecodeError, json.dumps, {b('\xa4'): 42}) - self.assertEqual(json.dumps({b('\xa4'): 42}, encoding='iso-8859-1'), - r'{"\u00a4": 42}') - self.assertEqual(json.dumps({b('\xa4'): 42}, encoding='iso-8859-15'), - r'{"\u20ac": 42}') - if PY3: - self.assertRaises(TypeError, json.dumps, {b('\xe2\x82\xac'): 42}, - encoding=None) - self.assertRaises(TypeError, json.dumps, {b('\xa4'): 42}, - encoding=None) - self.assertRaises(TypeError, json.dumps, {b('\xa4'): 42}, - encoding=None, default=decode_iso_8859_15) - self.assertEqual(json.dumps({b('\xa4'): 42}, encoding=None, - skipkeys=True), - r'{}') - else: - self.assertEqual(json.dumps({b('\xe2\x82\xac'): 42}, encoding=None), - r'{"\u20ac": 42}') - self.assertRaises(UnicodeDecodeError, json.dumps, {b('\xa4'): 42}, - encoding=None) - self.assertRaises(UnicodeDecodeError, json.dumps, {b('\xa4'): 42}, - encoding=None, default=decode_iso_8859_15) - self.assertRaises(UnicodeDecodeError, json.dumps, {b('\xa4'): 42}, - encoding=None, skipkeys=True) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_encode_basestring_ascii.py b/contrib/python/simplejson/py3/simplejson/tests/test_encode_basestring_ascii.py deleted file mode 100644 index 49706bfd978..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_encode_basestring_ascii.py +++ /dev/null @@ -1,47 +0,0 @@ -from unittest import TestCase - -import simplejson.encoder -from simplejson.compat import b - -CASES = [ - (u'/\\"\ucafe\ubabe\uab98\ufcde\ubcda\uef4a\x08\x0c\n\r\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?', '"/\\\\\\"\\ucafe\\ubabe\\uab98\\ufcde\\ubcda\\uef4a\\b\\f\\n\\r\\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?"'), - (u'\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), - (u'controls', '"controls"'), - (u'\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'), - (u'{"object with 1 member":["array with 1 element"]}', '"{\\"object with 1 member\\":[\\"array with 1 element\\"]}"'), - (u' s p a c e d ', '" s p a c e d "'), - (u'\U0001d120', '"\\ud834\\udd20"'), - (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'), - (b('\xce\xb1\xce\xa9'), '"\\u03b1\\u03a9"'), - (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'), - (b('\xce\xb1\xce\xa9'), '"\\u03b1\\u03a9"'), - (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'), - (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'), - (u"`1~!@#$%^&*()_+-={':[,]}|;.</>?", '"`1~!@#$%^&*()_+-={\':[,]}|;.</>?"'), - (u'\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'), - (u'\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), -] - -class TestEncodeBaseStringAscii(TestCase): - def test_py_encode_basestring_ascii(self): - self._test_encode_basestring_ascii(simplejson.encoder.py_encode_basestring_ascii) - - def test_c_encode_basestring_ascii(self): - if not simplejson.encoder.c_encode_basestring_ascii: - return - self._test_encode_basestring_ascii(simplejson.encoder.c_encode_basestring_ascii) - - def _test_encode_basestring_ascii(self, encode_basestring_ascii): - fname = encode_basestring_ascii.__name__ - for input_string, expect in CASES: - result = encode_basestring_ascii(input_string) - #self.assertEqual(result, expect, - # '{0!r} != {1!r} for {2}({3!r})'.format( - # result, expect, fname, input_string)) - self.assertEqual(result, expect, - '%r != %r for %s(%r)' % (result, expect, fname, input_string)) - - def test_sorted_dict(self): - items = [('one', 1), ('two', 2), ('three', 3), ('four', 4), ('five', 5)] - s = simplejson.dumps(dict(items), sort_keys=True) - self.assertEqual(s, '{"five": 5, "four": 4, "one": 1, "three": 3, "two": 2}') diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_encode_for_html.py b/contrib/python/simplejson/py3/simplejson/tests/test_encode_for_html.py deleted file mode 100644 index 3a840aa48cb..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_encode_for_html.py +++ /dev/null @@ -1,38 +0,0 @@ -import unittest - -import simplejson as json - -class TestEncodeForHTML(unittest.TestCase): - - def setUp(self): - self.decoder = json.JSONDecoder() - self.encoder = json.JSONEncoderForHTML() - self.non_ascii_encoder = json.JSONEncoderForHTML(ensure_ascii=False) - - def test_basic_encode(self): - self.assertEqual(r'"\u0026"', self.encoder.encode('&')) - self.assertEqual(r'"\u003c"', self.encoder.encode('<')) - self.assertEqual(r'"\u003e"', self.encoder.encode('>')) - self.assertEqual(r'"\u2028"', self.encoder.encode(u'\u2028')) - - def test_non_ascii_basic_encode(self): - self.assertEqual(r'"\u0026"', self.non_ascii_encoder.encode('&')) - self.assertEqual(r'"\u003c"', self.non_ascii_encoder.encode('<')) - self.assertEqual(r'"\u003e"', self.non_ascii_encoder.encode('>')) - self.assertEqual(r'"\u2028"', self.non_ascii_encoder.encode(u'\u2028')) - - def test_basic_roundtrip(self): - for char in '&<>': - self.assertEqual( - char, self.decoder.decode( - self.encoder.encode(char))) - - def test_prevent_script_breakout(self): - bad_string = '</script><script>alert("gotcha")</script>' - self.assertEqual( - r'"\u003c/script\u003e\u003cscript\u003e' - r'alert(\"gotcha\")\u003c/script\u003e"', - self.encoder.encode(bad_string)) - self.assertEqual( - bad_string, self.decoder.decode( - self.encoder.encode(bad_string))) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_errors.py b/contrib/python/simplejson/py3/simplejson/tests/test_errors.py deleted file mode 100644 index d5738254541..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_errors.py +++ /dev/null @@ -1,68 +0,0 @@ -import sys, pickle -from unittest import TestCase - -import simplejson as json -from simplejson.compat import text_type, b - -class TestErrors(TestCase): - def test_string_keys_error(self): - data = [{'a': 'A', 'b': (2, 4), 'c': 3.0, ('d',): 'D tuple'}] - try: - json.dumps(data) - except TypeError: - err = sys.exc_info()[1] - else: - self.fail('Expected TypeError') - self.assertEqual(str(err), - 'keys must be str, int, float, bool or None, not tuple') - - def test_not_serializable(self): - try: - json.dumps(json) - except TypeError: - err = sys.exc_info()[1] - else: - self.fail('Expected TypeError') - self.assertEqual(str(err), - 'Object of type module is not JSON serializable') - - def test_decode_error(self): - err = None - try: - json.loads('{}\na\nb') - except json.JSONDecodeError: - err = sys.exc_info()[1] - else: - self.fail('Expected JSONDecodeError') - self.assertEqual(err.lineno, 2) - self.assertEqual(err.colno, 1) - self.assertEqual(err.endlineno, 3) - self.assertEqual(err.endcolno, 2) - - def test_scan_error(self): - err = None - for t in (text_type, b): - try: - json.loads(t('{"asdf": "')) - except json.JSONDecodeError: - err = sys.exc_info()[1] - else: - self.fail('Expected JSONDecodeError') - self.assertEqual(err.lineno, 1) - self.assertEqual(err.colno, 10) - - def test_error_is_pickable(self): - err = None - try: - json.loads('{}\na\nb') - except json.JSONDecodeError: - err = sys.exc_info()[1] - else: - self.fail('Expected JSONDecodeError') - s = pickle.dumps(err) - e = pickle.loads(s) - - self.assertEqual(err.msg, e.msg) - self.assertEqual(err.doc, e.doc) - self.assertEqual(err.pos, e.pos) - self.assertEqual(err.end, e.end) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_fail.py b/contrib/python/simplejson/py3/simplejson/tests/test_fail.py deleted file mode 100644 index 5f9a8f698cf..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_fail.py +++ /dev/null @@ -1,178 +0,0 @@ -import sys -from unittest import TestCase - -import simplejson as json - -# 2007-10-05 -JSONDOCS = [ - # http://json.org/JSON_checker/test/fail1.json - '"A JSON payload should be an object or array, not a string."', - # http://json.org/JSON_checker/test/fail2.json - '["Unclosed array"', - # http://json.org/JSON_checker/test/fail3.json - '{unquoted_key: "keys must be quoted"}', - # http://json.org/JSON_checker/test/fail4.json - '["extra comma",]', - # http://json.org/JSON_checker/test/fail5.json - '["double extra comma",,]', - # http://json.org/JSON_checker/test/fail6.json - '[ , "<-- missing value"]', - # http://json.org/JSON_checker/test/fail7.json - '["Comma after the close"],', - # http://json.org/JSON_checker/test/fail8.json - '["Extra close"]]', - # http://json.org/JSON_checker/test/fail9.json - '{"Extra comma": true,}', - # http://json.org/JSON_checker/test/fail10.json - '{"Extra value after close": true} "misplaced quoted value"', - # http://json.org/JSON_checker/test/fail11.json - '{"Illegal expression": 1 + 2}', - # http://json.org/JSON_checker/test/fail12.json - '{"Illegal invocation": alert()}', - # http://json.org/JSON_checker/test/fail13.json - '{"Numbers cannot have leading zeroes": 013}', - # http://json.org/JSON_checker/test/fail14.json - '{"Numbers cannot be hex": 0x14}', - # http://json.org/JSON_checker/test/fail15.json - '["Illegal backslash escape: \\x15"]', - # http://json.org/JSON_checker/test/fail16.json - '[\\naked]', - # http://json.org/JSON_checker/test/fail17.json - '["Illegal backslash escape: \\017"]', - # http://json.org/JSON_checker/test/fail18.json - '[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]', - # http://json.org/JSON_checker/test/fail19.json - '{"Missing colon" null}', - # http://json.org/JSON_checker/test/fail20.json - '{"Double colon":: null}', - # http://json.org/JSON_checker/test/fail21.json - '{"Comma instead of colon", null}', - # http://json.org/JSON_checker/test/fail22.json - '["Colon instead of comma": false]', - # http://json.org/JSON_checker/test/fail23.json - '["Bad value", truth]', - # http://json.org/JSON_checker/test/fail24.json - "['single quote']", - # http://json.org/JSON_checker/test/fail25.json - '["\ttab\tcharacter\tin\tstring\t"]', - # http://json.org/JSON_checker/test/fail26.json - '["tab\\ character\\ in\\ string\\ "]', - # http://json.org/JSON_checker/test/fail27.json - '["line\nbreak"]', - # http://json.org/JSON_checker/test/fail28.json - '["line\\\nbreak"]', - # http://json.org/JSON_checker/test/fail29.json - '[0e]', - # http://json.org/JSON_checker/test/fail30.json - '[0e+]', - # http://json.org/JSON_checker/test/fail31.json - '[0e+-1]', - # http://json.org/JSON_checker/test/fail32.json - '{"Comma instead if closing brace": true,', - # http://json.org/JSON_checker/test/fail33.json - '["mismatch"}', - # http://code.google.com/p/simplejson/issues/detail?id=3 - u'["A\u001FZ control characters in string"]', - # misc based on coverage - '{', - '{]', - '{"foo": "bar"]', - '{"foo": "bar"', - 'nul', - 'nulx', - '-', - '-x', - '-e', - '-e0', - '-Infinite', - '-Inf', - 'Infinit', - 'Infinite', - 'NaM', - 'NuN', - 'falsy', - 'fal', - 'trug', - 'tru', - '1e', - '1ex', - '1e-', - '1e-x', -] - -SKIPS = { - 1: "why not have a string payload?", - 18: "spec doesn't specify any nesting limitations", -} - -class TestFail(TestCase): - def test_failures(self): - for idx, doc in enumerate(JSONDOCS): - idx = idx + 1 - if idx in SKIPS: - json.loads(doc) - continue - try: - json.loads(doc) - except json.JSONDecodeError: - pass - else: - self.fail("Expected failure for fail%d.json: %r" % (idx, doc)) - - def test_array_decoder_issue46(self): - # http://code.google.com/p/simplejson/issues/detail?id=46 - for doc in [u'[,]', '[,]']: - try: - json.loads(doc) - except json.JSONDecodeError: - e = sys.exc_info()[1] - self.assertEqual(e.pos, 1) - self.assertEqual(e.lineno, 1) - self.assertEqual(e.colno, 2) - except Exception: - e = sys.exc_info()[1] - self.fail("Unexpected exception raised %r %s" % (e, e)) - else: - self.fail("Unexpected success parsing '[,]'") - - def test_truncated_input(self): - test_cases = [ - ('', 'Expecting value', 0), - ('[', "Expecting value or ']'", 1), - ('[42', "Expecting ',' delimiter", 3), - ('[42,', 'Expecting value', 4), - ('["', 'Unterminated string starting at', 1), - ('["spam', 'Unterminated string starting at', 1), - ('["spam"', "Expecting ',' delimiter", 7), - ('["spam",', 'Expecting value', 8), - ('{', "Expecting property name enclosed in double quotes or '}'", 1), - ('{"', 'Unterminated string starting at', 1), - ('{"spam', 'Unterminated string starting at', 1), - ('{"spam"', "Expecting ':' delimiter", 7), - ('{"spam":', 'Expecting value', 8), - ('{"spam":42', "Expecting ',' delimiter", 10), - ('{"spam":42,', 'Expecting property name enclosed in double quotes', - 11), - ('"', 'Unterminated string starting at', 0), - ('"spam', 'Unterminated string starting at', 0), - ('[,', "Expecting value", 1), - ('--', 'Expecting value', 0), - ('"\x18d', "Invalid control character %r", 1), - ] - for data, msg, idx in test_cases: - try: - json.loads(data) - except json.JSONDecodeError: - e = sys.exc_info()[1] - self.assertEqual( - e.msg[:len(msg)], - msg, - "%r doesn't start with %r for %r" % (e.msg, msg, data)) - self.assertEqual( - e.pos, idx, - "pos %r != %r for %r" % (e.pos, idx, data)) - except Exception: - e = sys.exc_info()[1] - self.fail("Unexpected exception raised %r %s" % (e, e)) - else: - self.fail("Unexpected success parsing '%r'" % (data,)) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_float.py b/contrib/python/simplejson/py3/simplejson/tests/test_float.py deleted file mode 100644 index fed1bd97cd6..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_float.py +++ /dev/null @@ -1,46 +0,0 @@ -import sys -import math -from unittest import TestCase -from simplejson.compat import long_type, text_type -import simplejson as json -from simplejson.decoder import NaN, PosInf, NegInf - -class TestFloat(TestCase): - def test_degenerates_allow(self): - for inf in (PosInf, NegInf): - self.assertEqual(json.loads(json.dumps(inf, allow_nan=True), allow_nan=True), inf) - # Python 2.5 doesn't have math.isnan - nan = json.loads(json.dumps(NaN, allow_nan=True), allow_nan=True) - self.assertTrue((0 + nan) != nan) - - def test_degenerates_ignore(self): - for f in (PosInf, NegInf, NaN): - self.assertEqual(json.loads(json.dumps(f, ignore_nan=True)), None) - - def test_degenerates_deny(self): - for f in (PosInf, NegInf, NaN): - self.assertRaises(ValueError, json.dumps, f, allow_nan=False) - for s in ('Infinity', '-Infinity', 'NaN'): - self.assertRaises(ValueError, json.loads, s, allow_nan=False) - self.assertRaises(ValueError, json.loads, s) - - def test_floats(self): - for num in [1617161771.7650001, math.pi, math.pi**100, - math.pi**-100, 3.1]: - self.assertEqual(float(json.dumps(num)), num) - self.assertEqual(json.loads(json.dumps(num)), num) - self.assertEqual(json.loads(text_type(json.dumps(num))), num) - - def test_ints(self): - for num in [1, long_type(1), 1<<32, 1<<64]: - self.assertEqual(json.dumps(num), str(num)) - self.assertEqual(int(json.dumps(num)), num) - self.assertEqual(json.loads(json.dumps(num)), num) - self.assertEqual(json.loads(text_type(json.dumps(num))), num) - - def test_float_range(self): - try: - float_range = [sys.float_info.min, sys.float_info.max] - except AttributeError: - float_range = [2.2250738585072014e-308, 1.7976931348623157e+308] - self.assertEqual(json.loads(json.dumps(float_range)), float_range) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_for_json.py b/contrib/python/simplejson/py3/simplejson/tests/test_for_json.py deleted file mode 100644 index b791b883b04..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_for_json.py +++ /dev/null @@ -1,97 +0,0 @@ -import unittest -import simplejson as json - - -class ForJson(object): - def for_json(self): - return {'for_json': 1} - - -class NestedForJson(object): - def for_json(self): - return {'nested': ForJson()} - - -class ForJsonList(object): - def for_json(self): - return ['list'] - - -class DictForJson(dict): - def for_json(self): - return {'alpha': 1} - - -class ListForJson(list): - def for_json(self): - return ['list'] - - -class TestForJson(unittest.TestCase): - def assertRoundTrip(self, obj, other, for_json=True): - if for_json is None: - # None will use the default - s = json.dumps(obj) - else: - s = json.dumps(obj, for_json=for_json) - self.assertEqual( - json.loads(s), - other) - - def test_for_json_encodes_stand_alone_object(self): - self.assertRoundTrip( - ForJson(), - ForJson().for_json()) - - def test_for_json_encodes_object_nested_in_dict(self): - self.assertRoundTrip( - {'hooray': ForJson()}, - {'hooray': ForJson().for_json()}) - - def test_for_json_encodes_object_nested_in_list_within_dict(self): - self.assertRoundTrip( - {'list': [0, ForJson(), 2, 3]}, - {'list': [0, ForJson().for_json(), 2, 3]}) - - def test_for_json_encodes_object_nested_within_object(self): - self.assertRoundTrip( - NestedForJson(), - {'nested': {'for_json': 1}}) - - def test_for_json_encodes_list(self): - self.assertRoundTrip( - ForJsonList(), - ForJsonList().for_json()) - - def test_for_json_encodes_list_within_object(self): - self.assertRoundTrip( - {'nested': ForJsonList()}, - {'nested': ForJsonList().for_json()}) - - def test_for_json_encodes_dict_subclass(self): - self.assertRoundTrip( - DictForJson(a=1), - DictForJson(a=1).for_json()) - - def test_for_json_encodes_list_subclass(self): - self.assertRoundTrip( - ListForJson(['l']), - ListForJson(['l']).for_json()) - - def test_for_json_ignored_if_not_true_with_dict_subclass(self): - for for_json in (None, False): - self.assertRoundTrip( - DictForJson(a=1), - {'a': 1}, - for_json=for_json) - - def test_for_json_ignored_if_not_true_with_list_subclass(self): - for for_json in (None, False): - self.assertRoundTrip( - ListForJson(['l']), - ['l'], - for_json=for_json) - - def test_raises_typeerror_if_for_json_not_true_with_object(self): - self.assertRaises(TypeError, json.dumps, ForJson()) - self.assertRaises(TypeError, json.dumps, ForJson(), for_json=False) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_indent.py b/contrib/python/simplejson/py3/simplejson/tests/test_indent.py deleted file mode 100644 index cea25a575ed..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_indent.py +++ /dev/null @@ -1,86 +0,0 @@ -from unittest import TestCase -import textwrap - -import simplejson as json -from simplejson.compat import StringIO - -class TestIndent(TestCase): - def test_indent(self): - h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', - 'i-vhbjkhnth', - {'nifty': 87}, {'field': 'yes', 'morefield': False} ] - - expect = textwrap.dedent("""\ - [ - \t[ - \t\t"blorpie" - \t], - \t[ - \t\t"whoops" - \t], - \t[], - \t"d-shtaeou", - \t"d-nthiouh", - \t"i-vhbjkhnth", - \t{ - \t\t"nifty": 87 - \t}, - \t{ - \t\t"field": "yes", - \t\t"morefield": false - \t} - ]""") - - - d1 = json.dumps(h) - d2 = json.dumps(h, indent='\t', sort_keys=True, separators=(',', ': ')) - d3 = json.dumps(h, indent=' ', sort_keys=True, separators=(',', ': ')) - d4 = json.dumps(h, indent=2, sort_keys=True, separators=(',', ': ')) - - h1 = json.loads(d1) - h2 = json.loads(d2) - h3 = json.loads(d3) - h4 = json.loads(d4) - - self.assertEqual(h1, h) - self.assertEqual(h2, h) - self.assertEqual(h3, h) - self.assertEqual(h4, h) - self.assertEqual(d3, expect.replace('\t', ' ')) - self.assertEqual(d4, expect.replace('\t', ' ')) - # NOTE: Python 2.4 textwrap.dedent converts tabs to spaces, - # so the following is expected to fail. Python 2.4 is not a - # supported platform in simplejson 2.1.0+. - self.assertEqual(d2, expect) - - def test_indent0(self): - h = {3: 1} - def check(indent, expected): - d1 = json.dumps(h, indent=indent) - self.assertEqual(d1, expected) - - sio = StringIO() - json.dump(h, sio, indent=indent) - self.assertEqual(sio.getvalue(), expected) - - # indent=0 should emit newlines - check(0, '{\n"3": 1\n}') - # indent=None is more compact - check(None, '{"3": 1}') - - def test_separators(self): - lst = [1,2,3,4] - expect = '[\n1,\n2,\n3,\n4\n]' - expect_spaces = '[\n1, \n2, \n3, \n4\n]' - # Ensure that separators still works - self.assertEqual( - expect_spaces, - json.dumps(lst, indent=0, separators=(', ', ': '))) - # Force the new defaults - self.assertEqual( - expect, - json.dumps(lst, indent=0, separators=(',', ': '))) - # Added in 2.1.4 - self.assertEqual( - expect, - json.dumps(lst, indent=0)) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_item_sort_key.py b/contrib/python/simplejson/py3/simplejson/tests/test_item_sort_key.py deleted file mode 100644 index 98971b86ac7..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_item_sort_key.py +++ /dev/null @@ -1,27 +0,0 @@ -from unittest import TestCase - -import simplejson as json -from operator import itemgetter - -class TestItemSortKey(TestCase): - def test_simple_first(self): - a = {'a': 1, 'c': 5, 'jack': 'jill', 'pick': 'axe', 'array': [1, 5, 6, 9], 'tuple': (83, 12, 3), 'crate': 'dog', 'zeak': 'oh'} - self.assertEqual( - '{"a": 1, "c": 5, "crate": "dog", "jack": "jill", "pick": "axe", "zeak": "oh", "array": [1, 5, 6, 9], "tuple": [83, 12, 3]}', - json.dumps(a, item_sort_key=json.simple_first)) - - def test_case(self): - a = {'a': 1, 'c': 5, 'Jack': 'jill', 'pick': 'axe', 'Array': [1, 5, 6, 9], 'tuple': (83, 12, 3), 'crate': 'dog', 'zeak': 'oh'} - self.assertEqual( - '{"Array": [1, 5, 6, 9], "Jack": "jill", "a": 1, "c": 5, "crate": "dog", "pick": "axe", "tuple": [83, 12, 3], "zeak": "oh"}', - json.dumps(a, item_sort_key=itemgetter(0))) - self.assertEqual( - '{"a": 1, "Array": [1, 5, 6, 9], "c": 5, "crate": "dog", "Jack": "jill", "pick": "axe", "tuple": [83, 12, 3], "zeak": "oh"}', - json.dumps(a, item_sort_key=lambda kv: kv[0].lower())) - - def test_item_sort_key_value(self): - # https://github.com/simplejson/simplejson/issues/173 - a = {'a': 1, 'b': 0} - self.assertEqual( - '{"b": 0, "a": 1}', - json.dumps(a, item_sort_key=lambda kv: kv[1])) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_iterable.py b/contrib/python/simplejson/py3/simplejson/tests/test_iterable.py deleted file mode 100644 index 35d3e759111..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_iterable.py +++ /dev/null @@ -1,31 +0,0 @@ -import unittest -from simplejson.compat import StringIO - -import simplejson as json - -def iter_dumps(obj, **kw): - return ''.join(json.JSONEncoder(**kw).iterencode(obj)) - -def sio_dump(obj, **kw): - sio = StringIO() - json.dumps(obj, **kw) - return sio.getvalue() - -class TestIterable(unittest.TestCase): - def test_iterable(self): - for l in ([], [1], [1, 2], [1, 2, 3]): - for opts in [{}, {'indent': 2}]: - for dumps in (json.dumps, iter_dumps, sio_dump): - expect = dumps(l, **opts) - default_expect = dumps(sum(l), **opts) - # Default is False - self.assertRaises(TypeError, dumps, iter(l), **opts) - self.assertRaises(TypeError, dumps, iter(l), iterable_as_array=False, **opts) - self.assertEqual(expect, dumps(iter(l), iterable_as_array=True, **opts)) - # Ensure that the "default" gets called - self.assertEqual(default_expect, dumps(iter(l), default=sum, **opts)) - self.assertEqual(default_expect, dumps(iter(l), iterable_as_array=False, default=sum, **opts)) - # Ensure that the "default" does not get called - self.assertEqual( - expect, - dumps(iter(l), iterable_as_array=True, default=sum, **opts)) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_namedtuple.py b/contrib/python/simplejson/py3/simplejson/tests/test_namedtuple.py deleted file mode 100644 index cc0f8aa1d7d..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_namedtuple.py +++ /dev/null @@ -1,174 +0,0 @@ -from __future__ import absolute_import -import unittest -import simplejson as json -from simplejson.compat import StringIO - -try: - from unittest import mock -except ImportError: - mock = None - -try: - from collections import namedtuple -except ImportError: - class Value(tuple): - def __new__(cls, *args): - return tuple.__new__(cls, args) - - def _asdict(self): - return {'value': self[0]} - class Point(tuple): - def __new__(cls, *args): - return tuple.__new__(cls, args) - - def _asdict(self): - return {'x': self[0], 'y': self[1]} -else: - Value = namedtuple('Value', ['value']) - Point = namedtuple('Point', ['x', 'y']) - -class DuckValue(object): - def __init__(self, *args): - self.value = Value(*args) - - def _asdict(self): - return self.value._asdict() - -class DuckPoint(object): - def __init__(self, *args): - self.point = Point(*args) - - def _asdict(self): - return self.point._asdict() - -class DeadDuck(object): - _asdict = None - -class DeadDict(dict): - _asdict = None - -CONSTRUCTORS = [ - lambda v: v, - lambda v: [v], - lambda v: [{'key': v}], -] - -class TestNamedTuple(unittest.TestCase): - def test_namedtuple_dumps(self): - for v in [Value(1), Point(1, 2), DuckValue(1), DuckPoint(1, 2)]: - d = v._asdict() - self.assertEqual(d, json.loads(json.dumps(v))) - self.assertEqual( - d, - json.loads(json.dumps(v, namedtuple_as_object=True))) - self.assertEqual(d, json.loads(json.dumps(v, tuple_as_array=False))) - self.assertEqual( - d, - json.loads(json.dumps(v, namedtuple_as_object=True, - tuple_as_array=False))) - - def test_namedtuple_dumps_false(self): - for v in [Value(1), Point(1, 2)]: - l = list(v) - self.assertEqual( - l, - json.loads(json.dumps(v, namedtuple_as_object=False))) - self.assertRaises(TypeError, json.dumps, v, - tuple_as_array=False, namedtuple_as_object=False) - - def test_namedtuple_dump(self): - for v in [Value(1), Point(1, 2), DuckValue(1), DuckPoint(1, 2)]: - d = v._asdict() - sio = StringIO() - json.dump(v, sio) - self.assertEqual(d, json.loads(sio.getvalue())) - sio = StringIO() - json.dump(v, sio, namedtuple_as_object=True) - self.assertEqual( - d, - json.loads(sio.getvalue())) - sio = StringIO() - json.dump(v, sio, tuple_as_array=False) - self.assertEqual(d, json.loads(sio.getvalue())) - sio = StringIO() - json.dump(v, sio, namedtuple_as_object=True, - tuple_as_array=False) - self.assertEqual( - d, - json.loads(sio.getvalue())) - - def test_namedtuple_dump_false(self): - for v in [Value(1), Point(1, 2)]: - l = list(v) - sio = StringIO() - json.dump(v, sio, namedtuple_as_object=False) - self.assertEqual( - l, - json.loads(sio.getvalue())) - self.assertRaises(TypeError, json.dump, v, StringIO(), - tuple_as_array=False, namedtuple_as_object=False) - - def test_asdict_not_callable_dump(self): - for f in CONSTRUCTORS: - self.assertRaises( - TypeError, - json.dump, - f(DeadDuck()), - StringIO(), - namedtuple_as_object=True - ) - sio = StringIO() - json.dump(f(DeadDict()), sio, namedtuple_as_object=True) - self.assertEqual( - json.dumps(f({})), - sio.getvalue()) - self.assertRaises( - TypeError, - json.dump, - f(Value), - StringIO(), - namedtuple_as_object=True - ) - - def test_asdict_not_callable_dumps(self): - for f in CONSTRUCTORS: - self.assertRaises(TypeError, - json.dumps, f(DeadDuck()), namedtuple_as_object=True) - self.assertRaises( - TypeError, - json.dumps, - f(Value), - namedtuple_as_object=True - ) - self.assertEqual( - json.dumps(f({})), - json.dumps(f(DeadDict()), namedtuple_as_object=True)) - - def test_asdict_unbound_method_dumps(self): - for f in CONSTRUCTORS: - self.assertEqual( - json.dumps(f(Value), default=lambda v: v.__name__), - json.dumps(f(Value.__name__)) - ) - - def test_asdict_does_not_return_dict(self): - if not mock: - if hasattr(unittest, "SkipTest"): - raise unittest.SkipTest("unittest.mock required") - else: - print("unittest.mock not available") - return - fake = mock.Mock() - self.assertTrue(hasattr(fake, '_asdict')) - self.assertTrue(callable(fake._asdict)) - self.assertFalse(isinstance(fake._asdict(), dict)) - # https://github.com/simplejson/simplejson/pull/284 - # when running under a debug build of CPython (COPTS=-UNDEBUG) - # a C assertion could fire due to an unchecked error of an PyDict - # API call on a non-dict internally in _speedups.c. Without a debug - # build of CPython this test likely passes either way despite the - # potential for internal data corruption. Getting it to crash in - # a debug build is not always easy either as it requires an - # assert(!PyErr_Occurred()) that could fire later on. - with self.assertRaises(TypeError): - json.dumps({23: fake}, namedtuple_as_object=True, for_json=False) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_pass1.py b/contrib/python/simplejson/py3/simplejson/tests/test_pass1.py deleted file mode 100644 index f0b5b10e761..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_pass1.py +++ /dev/null @@ -1,71 +0,0 @@ -from unittest import TestCase - -import simplejson as json - -# from http://json.org/JSON_checker/test/pass1.json -JSON = r''' -[ - "JSON Test Pattern pass1", - {"object with 1 member":["array with 1 element"]}, - {}, - [], - -42, - true, - false, - null, - { - "integer": 1234567890, - "real": -9876.543210, - "e": 0.123456789e-12, - "E": 1.234567890E+34, - "": 23456789012E66, - "zero": 0, - "one": 1, - "space": " ", - "quote": "\"", - "backslash": "\\", - "controls": "\b\f\n\r\t", - "slash": "/ & \/", - "alpha": "abcdefghijklmnopqrstuvwyz", - "ALPHA": "ABCDEFGHIJKLMNOPQRSTUVWYZ", - "digit": "0123456789", - "special": "`1~!@#$%^&*()_+-={':[,]}|;.</>?", - "hex": "\u0123\u4567\u89AB\uCDEF\uabcd\uef4A", - "true": true, - "false": false, - "null": null, - "array":[ ], - "object":{ }, - "address": "50 St. James Street", - "url": "http://www.JSON.org/", - "comment": "// /* <!-- --", - "# -- --> */": " ", - " s p a c e d " :[1,2 , 3 - -, - -4 , 5 , 6 ,7 ],"compact": [1,2,3,4,5,6,7], - "jsontext": "{\"object with 1 member\":[\"array with 1 element\"]}", - "quotes": "" \u0022 %22 0x22 034 "", - "\/\\\"\uCAFE\uBABE\uAB98\uFCDE\ubcda\uef4A\b\f\n\r\t`1~!@#$%^&*()_+-=[]{}|;:',./<>?" -: "A key can be any string" - }, - 0.5 ,98.6 -, -99.44 -, - -1066, -1e1, -0.1e1, -1e-1, -1e00,2e+00,2e-00 -,"rosebud"] -''' - -class TestPass1(TestCase): - def test_parse(self): - # test in/out equivalence and parsing - res = json.loads(JSON) - out = json.dumps(res) - self.assertEqual(res, json.loads(out)) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_pass2.py b/contrib/python/simplejson/py3/simplejson/tests/test_pass2.py deleted file mode 100644 index 5d812b3bbe2..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_pass2.py +++ /dev/null @@ -1,14 +0,0 @@ -from unittest import TestCase -import simplejson as json - -# from http://json.org/JSON_checker/test/pass2.json -JSON = r''' -[[[[[[[[[[[[[[[[[[["Not too deep"]]]]]]]]]]]]]]]]]]] -''' - -class TestPass2(TestCase): - def test_parse(self): - # test in/out equivalence and parsing - res = json.loads(JSON) - out = json.dumps(res) - self.assertEqual(res, json.loads(out)) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_pass3.py b/contrib/python/simplejson/py3/simplejson/tests/test_pass3.py deleted file mode 100644 index 821d60b22cc..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_pass3.py +++ /dev/null @@ -1,20 +0,0 @@ -from unittest import TestCase - -import simplejson as json - -# from http://json.org/JSON_checker/test/pass3.json -JSON = r''' -{ - "JSON Test Pattern pass3": { - "The outermost value": "must be an object or array.", - "In this test": "It is an object." - } -} -''' - -class TestPass3(TestCase): - def test_parse(self): - # test in/out equivalence and parsing - res = json.loads(JSON) - out = json.dumps(res) - self.assertEqual(res, json.loads(out)) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_raw_json.py b/contrib/python/simplejson/py3/simplejson/tests/test_raw_json.py deleted file mode 100644 index 1dfcc2c5f95..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_raw_json.py +++ /dev/null @@ -1,47 +0,0 @@ -import unittest -import simplejson as json - -dct1 = { - 'key1': 'value1' -} - -dct2 = { - 'key2': 'value2', - 'd1': dct1 -} - -dct3 = { - 'key2': 'value2', - 'd1': json.dumps(dct1) -} - -dct4 = { - 'key2': 'value2', - 'd1': json.RawJSON(json.dumps(dct1)) -} - - -class TestRawJson(unittest.TestCase): - - def test_normal_str(self): - self.assertNotEqual(json.dumps(dct2), json.dumps(dct3)) - - def test_raw_json_str(self): - self.assertEqual(json.dumps(dct2), json.dumps(dct4)) - self.assertEqual(dct2, json.loads(json.dumps(dct4))) - - def test_list(self): - self.assertEqual( - json.dumps([dct2]), - json.dumps([json.RawJSON(json.dumps(dct2))])) - self.assertEqual( - [dct2], - json.loads(json.dumps([json.RawJSON(json.dumps(dct2))]))) - - def test_direct(self): - self.assertEqual( - json.dumps(dct2), - json.dumps(json.RawJSON(json.dumps(dct2)))) - self.assertEqual( - dct2, - json.loads(json.dumps(json.RawJSON(json.dumps(dct2))))) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_recursion.py b/contrib/python/simplejson/py3/simplejson/tests/test_recursion.py deleted file mode 100644 index 662eb667ece..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_recursion.py +++ /dev/null @@ -1,67 +0,0 @@ -from unittest import TestCase - -import simplejson as json - -class JSONTestObject: - pass - - -class RecursiveJSONEncoder(json.JSONEncoder): - recurse = False - def default(self, o): - if o is JSONTestObject: - if self.recurse: - return [JSONTestObject] - else: - return 'JSONTestObject' - return json.JSONEncoder.default(o) - - -class TestRecursion(TestCase): - def test_listrecursion(self): - x = [] - x.append(x) - try: - json.dumps(x) - except ValueError: - pass - else: - self.fail("didn't raise ValueError on list recursion") - x = [] - y = [x] - x.append(y) - try: - json.dumps(x) - except ValueError: - pass - else: - self.fail("didn't raise ValueError on alternating list recursion") - y = [] - x = [y, y] - # ensure that the marker is cleared - json.dumps(x) - - def test_dictrecursion(self): - x = {} - x["test"] = x - try: - json.dumps(x) - except ValueError: - pass - else: - self.fail("didn't raise ValueError on dict recursion") - x = {} - y = {"a": x, "b": x} - # ensure that the marker is cleared - json.dumps(y) - - def test_defaultrecursion(self): - enc = RecursiveJSONEncoder() - self.assertEqual(enc.encode(JSONTestObject), '"JSONTestObject"') - enc.recurse = True - try: - enc.encode(JSONTestObject) - except ValueError: - pass - else: - self.fail("didn't raise ValueError on default recursion") diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_scanstring.py b/contrib/python/simplejson/py3/simplejson/tests/test_scanstring.py deleted file mode 100644 index 1f544834209..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_scanstring.py +++ /dev/null @@ -1,200 +0,0 @@ -import sys -from unittest import TestCase - -import simplejson as json -import simplejson.decoder -from simplejson.compat import b, PY3 - -class TestScanString(TestCase): - # The bytes type is intentionally not used in most of these tests - # under Python 3 because the decoder immediately coerces to str before - # calling scanstring. In Python 2 we are testing the code paths - # for both unicode and str. - # - # The reason this is done is because Python 3 would require - # entirely different code paths for parsing bytes and str. - # - def test_py_scanstring(self): - self._test_scanstring(simplejson.decoder.py_scanstring) - - def test_c_scanstring(self): - if not simplejson.decoder.c_scanstring: - return - self._test_scanstring(simplejson.decoder.c_scanstring) - - self.assertTrue(isinstance(simplejson.decoder.c_scanstring('""', 0)[0], str)) - - def _test_scanstring(self, scanstring): - if sys.maxunicode == 65535: - self.assertEqual( - scanstring(u'"z\U0001d120x"', 1, None, True), - (u'z\U0001d120x', 6)) - else: - self.assertEqual( - scanstring(u'"z\U0001d120x"', 1, None, True), - (u'z\U0001d120x', 5)) - - self.assertEqual( - scanstring('"\\u007b"', 1, None, True), - (u'{', 8)) - - self.assertEqual( - scanstring('"A JSON payload should be an object or array, not a string."', 1, None, True), - (u'A JSON payload should be an object or array, not a string.', 60)) - - self.assertEqual( - scanstring('["Unclosed array"', 2, None, True), - (u'Unclosed array', 17)) - - self.assertEqual( - scanstring('["extra comma",]', 2, None, True), - (u'extra comma', 14)) - - self.assertEqual( - scanstring('["double extra comma",,]', 2, None, True), - (u'double extra comma', 21)) - - self.assertEqual( - scanstring('["Comma after the close"],', 2, None, True), - (u'Comma after the close', 24)) - - self.assertEqual( - scanstring('["Extra close"]]', 2, None, True), - (u'Extra close', 14)) - - self.assertEqual( - scanstring('{"Extra comma": true,}', 2, None, True), - (u'Extra comma', 14)) - - self.assertEqual( - scanstring('{"Extra value after close": true} "misplaced quoted value"', 2, None, True), - (u'Extra value after close', 26)) - - self.assertEqual( - scanstring('{"Illegal expression": 1 + 2}', 2, None, True), - (u'Illegal expression', 21)) - - self.assertEqual( - scanstring('{"Illegal invocation": alert()}', 2, None, True), - (u'Illegal invocation', 21)) - - self.assertEqual( - scanstring('{"Numbers cannot have leading zeroes": 013}', 2, None, True), - (u'Numbers cannot have leading zeroes', 37)) - - self.assertEqual( - scanstring('{"Numbers cannot be hex": 0x14}', 2, None, True), - (u'Numbers cannot be hex', 24)) - - self.assertEqual( - scanstring('[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]', 21, None, True), - (u'Too deep', 30)) - - self.assertEqual( - scanstring('{"Missing colon" null}', 2, None, True), - (u'Missing colon', 16)) - - self.assertEqual( - scanstring('{"Double colon":: null}', 2, None, True), - (u'Double colon', 15)) - - self.assertEqual( - scanstring('{"Comma instead of colon", null}', 2, None, True), - (u'Comma instead of colon', 25)) - - self.assertEqual( - scanstring('["Colon instead of comma": false]', 2, None, True), - (u'Colon instead of comma', 25)) - - self.assertEqual( - scanstring('["Bad value", truth]', 2, None, True), - (u'Bad value', 12)) - - for c in map(chr, range(0x00, 0x1f)): - self.assertEqual( - scanstring(c + '"', 0, None, False), - (c, 2)) - self.assertRaises( - ValueError, - scanstring, c + '"', 0, None, True) - - self.assertRaises(ValueError, scanstring, '', 0, None, True) - self.assertRaises(ValueError, scanstring, 'a', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\u', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\u0', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\u01', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\u012', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\u0123', 0, None, True) - if sys.maxunicode > 65535: - self.assertRaises(ValueError, - scanstring, '\\ud834\\u"', 0, None, True) - self.assertRaises(ValueError, - scanstring, '\\ud834\\x0123"', 0, None, True) - - self.assertRaises(json.JSONDecodeError, scanstring, '\\u-123"', 0, None, True) - # SJ-PT-23-01: Invalid Handling of Broken Unicode Escape Sequences - self.assertRaises(json.JSONDecodeError, scanstring, '\\u EDD"', 0, None, True) - - def test_issue3623(self): - self.assertRaises(ValueError, json.decoder.scanstring, "xxx", 1, - "xxx") - self.assertRaises(UnicodeDecodeError, - json.encoder.encode_basestring_ascii, b("xx\xff")) - - def test_overflow(self): - # Python 2.5 does not have maxsize, Python 3 does not have maxint - maxsize = getattr(sys, 'maxsize', getattr(sys, 'maxint', None)) - assert maxsize is not None - self.assertRaises(OverflowError, json.decoder.scanstring, "xxx", - maxsize + 1) - - def test_surrogates(self): - scanstring = json.decoder.scanstring - - def assertScan(given, expect, test_utf8=True): - givens = [given] - if not PY3 and test_utf8: - givens.append(given.encode('utf8')) - for given in givens: - (res, count) = scanstring(given, 1, None, True) - self.assertEqual(len(given), count) - self.assertEqual(res, expect) - - assertScan( - u'"z\\ud834\\u0079x"', - u'z\ud834yx') - assertScan( - u'"z\\ud834\\udd20x"', - u'z\U0001d120x') - assertScan( - u'"z\\ud834\\ud834\\udd20x"', - u'z\ud834\U0001d120x') - assertScan( - u'"z\\ud834x"', - u'z\ud834x') - assertScan( - u'"z\\udd20x"', - u'z\udd20x') - assertScan( - u'"z\ud834x"', - u'z\ud834x') - # It may look strange to join strings together, but Python is drunk. - # https://gist.github.com/etrepum/5538443 - assertScan( - u'"z\\ud834\udd20x12345"', - u''.join([u'z\ud834', u'\udd20x12345'])) - assertScan( - u'"z\ud834\\udd20x"', - u''.join([u'z\ud834', u'\udd20x'])) - # these have different behavior given UTF8 input, because the surrogate - # pair may be joined (in maxunicode > 65535 builds) - assertScan( - u''.join([u'"z\ud834', u'\udd20x"']), - u''.join([u'z\ud834', u'\udd20x']), - test_utf8=False) - - self.assertRaises(ValueError, - scanstring, u'"z\\ud83x"', 1, None, True) - self.assertRaises(ValueError, - scanstring, u'"z\\ud834\\udd2x"', 1, None, True) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_separators.py b/contrib/python/simplejson/py3/simplejson/tests/test_separators.py deleted file mode 100644 index 91b4d4fb6f3..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_separators.py +++ /dev/null @@ -1,42 +0,0 @@ -import textwrap -from unittest import TestCase - -import simplejson as json - - -class TestSeparators(TestCase): - def test_separators(self): - h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', 'i-vhbjkhnth', - {'nifty': 87}, {'field': 'yes', 'morefield': False} ] - - expect = textwrap.dedent("""\ - [ - [ - "blorpie" - ] , - [ - "whoops" - ] , - [] , - "d-shtaeou" , - "d-nthiouh" , - "i-vhbjkhnth" , - { - "nifty" : 87 - } , - { - "field" : "yes" , - "morefield" : false - } - ]""") - - - d1 = json.dumps(h) - d2 = json.dumps(h, indent=' ', sort_keys=True, separators=(' ,', ' : ')) - - h1 = json.loads(d1) - h2 = json.loads(d2) - - self.assertEqual(h1, h) - self.assertEqual(h2, h) - self.assertEqual(d2, expect) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_speedups.py b/contrib/python/simplejson/py3/simplejson/tests/test_speedups.py deleted file mode 100644 index 8b146df9a05..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_speedups.py +++ /dev/null @@ -1,114 +0,0 @@ -from __future__ import with_statement - -import sys -import unittest -from unittest import TestCase - -import simplejson -from simplejson import encoder, decoder, scanner -from simplejson.compat import PY3, long_type, b - - -def has_speedups(): - return encoder.c_make_encoder is not None - - -def skip_if_speedups_missing(func): - def wrapper(*args, **kwargs): - if not has_speedups(): - if hasattr(unittest, 'SkipTest'): - raise unittest.SkipTest("C Extension not available") - else: - sys.stdout.write("C Extension not available") - return - return func(*args, **kwargs) - - return wrapper - - -class BadBool: - def __bool__(self): - 1/0 - __nonzero__ = __bool__ - - -class TestDecode(TestCase): - @skip_if_speedups_missing - def test_make_scanner(self): - self.assertRaises(AttributeError, scanner.c_make_scanner, 1) - - @skip_if_speedups_missing - def test_bad_bool_args(self): - def test(value): - decoder.JSONDecoder(strict=BadBool()).decode(value) - self.assertRaises(ZeroDivisionError, test, '""') - self.assertRaises(ZeroDivisionError, test, '{}') - if not PY3: - self.assertRaises(ZeroDivisionError, test, u'""') - self.assertRaises(ZeroDivisionError, test, u'{}') - -class TestEncode(TestCase): - @skip_if_speedups_missing - def test_make_encoder(self): - self.assertRaises( - TypeError, - encoder.c_make_encoder, - None, - ("\xCD\x7D\x3D\x4E\x12\x4C\xF9\x79\xD7" - "\x52\xBA\x82\xF2\x27\x4A\x7D\xA0\xCA\x75"), - None - ) - - @skip_if_speedups_missing - def test_bad_str_encoder(self): - # Issue #31505: There shouldn't be an assertion failure in case - # c_make_encoder() receives a bad encoder() argument. - import decimal - def bad_encoder1(*args): - return None - enc = encoder.c_make_encoder( - None, lambda obj: str(obj), - bad_encoder1, None, ': ', ', ', - False, False, False, {}, False, False, False, - None, None, 'utf-8', False, False, decimal.Decimal, False) - self.assertRaises(TypeError, enc, 'spam', 4) - self.assertRaises(TypeError, enc, {'spam': 42}, 4) - - def bad_encoder2(*args): - 1/0 - enc = encoder.c_make_encoder( - None, lambda obj: str(obj), - bad_encoder2, None, ': ', ', ', - False, False, False, {}, False, False, False, - None, None, 'utf-8', False, False, decimal.Decimal, False) - self.assertRaises(ZeroDivisionError, enc, 'spam', 4) - - @skip_if_speedups_missing - def test_bad_bool_args(self): - def test(name): - encoder.JSONEncoder(**{name: BadBool()}).encode({}) - self.assertRaises(ZeroDivisionError, test, 'skipkeys') - self.assertRaises(ZeroDivisionError, test, 'ensure_ascii') - self.assertRaises(ZeroDivisionError, test, 'check_circular') - self.assertRaises(ZeroDivisionError, test, 'allow_nan') - self.assertRaises(ZeroDivisionError, test, 'sort_keys') - self.assertRaises(ZeroDivisionError, test, 'use_decimal') - self.assertRaises(ZeroDivisionError, test, 'namedtuple_as_object') - self.assertRaises(ZeroDivisionError, test, 'tuple_as_array') - self.assertRaises(ZeroDivisionError, test, 'bigint_as_string') - self.assertRaises(ZeroDivisionError, test, 'for_json') - self.assertRaises(ZeroDivisionError, test, 'ignore_nan') - self.assertRaises(ZeroDivisionError, test, 'iterable_as_array') - - @skip_if_speedups_missing - def test_int_as_string_bitcount_overflow(self): - long_count = long_type(2)**32+31 - def test(): - encoder.JSONEncoder(int_as_string_bitcount=long_count).encode(0) - self.assertRaises((TypeError, OverflowError), test) - - if PY3: - @skip_if_speedups_missing - def test_bad_encoding(self): - with self.assertRaises(UnicodeEncodeError): - encoder.JSONEncoder(encoding='\udcff').encode({b('key'): 123}) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_str_subclass.py b/contrib/python/simplejson/py3/simplejson/tests/test_str_subclass.py deleted file mode 100644 index b6c8351f264..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_str_subclass.py +++ /dev/null @@ -1,21 +0,0 @@ -from unittest import TestCase - -import simplejson -from simplejson.compat import text_type - -# Tests for issue demonstrated in https://github.com/simplejson/simplejson/issues/144 -class WonkyTextSubclass(text_type): - def __getslice__(self, start, end): - return self.__class__('not what you wanted!') - -class TestStrSubclass(TestCase): - def test_dump_load(self): - for s in ['', '"hello"', 'text', u'\u005c']: - self.assertEqual( - s, - simplejson.loads(simplejson.dumps(WonkyTextSubclass(s)))) - - self.assertEqual( - s, - simplejson.loads(simplejson.dumps(WonkyTextSubclass(s), - ensure_ascii=False))) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_subclass.py b/contrib/python/simplejson/py3/simplejson/tests/test_subclass.py deleted file mode 100644 index 2bae3b6309e..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_subclass.py +++ /dev/null @@ -1,37 +0,0 @@ -from unittest import TestCase -import simplejson as json - -from decimal import Decimal - -class AlternateInt(int): - def __repr__(self): - return 'invalid json' - __str__ = __repr__ - - -class AlternateFloat(float): - def __repr__(self): - return 'invalid json' - __str__ = __repr__ - - -# class AlternateDecimal(Decimal): -# def __repr__(self): -# return 'invalid json' - - -class TestSubclass(TestCase): - def test_int(self): - self.assertEqual(json.dumps(AlternateInt(1)), '1') - self.assertEqual(json.dumps(AlternateInt(-1)), '-1') - self.assertEqual(json.loads(json.dumps({AlternateInt(1): 1})), {'1': 1}) - - def test_float(self): - self.assertEqual(json.dumps(AlternateFloat(1.0)), '1.0') - self.assertEqual(json.dumps(AlternateFloat(-1.0)), '-1.0') - self.assertEqual(json.loads(json.dumps({AlternateFloat(1.0): 1})), {'1.0': 1}) - - # NOTE: Decimal subclasses are not supported as-is - # def test_decimal(self): - # self.assertEqual(json.dumps(AlternateDecimal('1.0')), '1.0') - # self.assertEqual(json.dumps(AlternateDecimal('-1.0')), '-1.0') diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_tool.py b/contrib/python/simplejson/py3/simplejson/tests/test_tool.py deleted file mode 100644 index 914bff8bca5..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_tool.py +++ /dev/null @@ -1,114 +0,0 @@ -from __future__ import with_statement -import os -import sys -import textwrap -import unittest -import subprocess -import tempfile -try: - # Python 3.x - from test.support import strip_python_stderr -except ImportError: - # Python 2.6+ - try: - from test.test_support import strip_python_stderr - except ImportError: - # Python 2.5 - import re - def strip_python_stderr(stderr): - return re.sub( - r"\[\d+ refs\]\r?\n?$".encode(), - "".encode(), - stderr).strip() - -def open_temp_file(): - if sys.version_info >= (2, 6): - file = tempfile.NamedTemporaryFile(delete=False) - filename = file.name - else: - fd, filename = tempfile.mkstemp() - file = os.fdopen(fd, 'w+b') - return file, filename - -class TestTool(unittest.TestCase): - data = """ - - [["blorpie"],[ "whoops" ] , [ - ],\t"d-shtaeou",\r"d-nthiouh", - "i-vhbjkhnth", {"nifty":87}, {"morefield" :\tfalse,"field" - :"yes"} ] - """ - - expect = textwrap.dedent("""\ - [ - [ - "blorpie" - ], - [ - "whoops" - ], - [], - "d-shtaeou", - "d-nthiouh", - "i-vhbjkhnth", - { - "nifty": 87 - }, - { - "field": "yes", - "morefield": false - } - ] - """) - - def runTool(self, args=None, data=None): - argv = [sys.executable, '-m', 'simplejson.tool'] - if args: - argv.extend(args) - proc = subprocess.Popen(argv, - stdin=subprocess.PIPE, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE) - out, err = proc.communicate(data) - self.assertEqual(strip_python_stderr(err), ''.encode()) - self.assertEqual(proc.returncode, 0) - return out.decode('utf8').splitlines() - - def test_stdin_stdout(self): - self.assertEqual( - self.runTool(data=self.data.encode()), - self.expect.splitlines()) - - def test_infile_stdout(self): - infile, infile_name = open_temp_file() - try: - infile.write(self.data.encode()) - infile.close() - self.assertEqual( - self.runTool(args=[infile_name]), - self.expect.splitlines()) - finally: - os.unlink(infile_name) - - def test_infile_outfile(self): - infile, infile_name = open_temp_file() - try: - infile.write(self.data.encode()) - infile.close() - # outfile will get overwritten by tool, so the delete - # may not work on some platforms. Do it manually. - outfile, outfile_name = open_temp_file() - try: - outfile.close() - self.assertEqual( - self.runTool(args=[infile_name, outfile_name]), - []) - with open(outfile_name, 'rb') as f: - self.assertEqual( - f.read().decode('utf8').splitlines(), - self.expect.splitlines() - ) - finally: - os.unlink(outfile_name) - finally: - os.unlink(infile_name) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_tuple.py b/contrib/python/simplejson/py3/simplejson/tests/test_tuple.py deleted file mode 100644 index 4ad7b0e861b..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_tuple.py +++ /dev/null @@ -1,47 +0,0 @@ -import unittest - -from simplejson.compat import StringIO -import simplejson as json - -class TestTuples(unittest.TestCase): - def test_tuple_array_dumps(self): - t = (1, 2, 3) - expect = json.dumps(list(t)) - # Default is True - self.assertEqual(expect, json.dumps(t)) - self.assertEqual(expect, json.dumps(t, tuple_as_array=True)) - self.assertRaises(TypeError, json.dumps, t, tuple_as_array=False) - # Ensure that the "default" does not get called - self.assertEqual(expect, json.dumps(t, default=repr)) - self.assertEqual(expect, json.dumps(t, tuple_as_array=True, - default=repr)) - # Ensure that the "default" gets called - self.assertEqual( - json.dumps(repr(t)), - json.dumps(t, tuple_as_array=False, default=repr)) - - def test_tuple_array_dump(self): - t = (1, 2, 3) - expect = json.dumps(list(t)) - # Default is True - sio = StringIO() - json.dump(t, sio) - self.assertEqual(expect, sio.getvalue()) - sio = StringIO() - json.dump(t, sio, tuple_as_array=True) - self.assertEqual(expect, sio.getvalue()) - self.assertRaises(TypeError, json.dump, t, StringIO(), - tuple_as_array=False) - # Ensure that the "default" does not get called - sio = StringIO() - json.dump(t, sio, default=repr) - self.assertEqual(expect, sio.getvalue()) - sio = StringIO() - json.dump(t, sio, tuple_as_array=True, default=repr) - self.assertEqual(expect, sio.getvalue()) - # Ensure that the "default" gets called - sio = StringIO() - json.dump(t, sio, tuple_as_array=False, default=repr) - self.assertEqual( - json.dumps(repr(t)), - sio.getvalue()) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_unicode.py b/contrib/python/simplejson/py3/simplejson/tests/test_unicode.py deleted file mode 100644 index 0c7b1a6c444..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_unicode.py +++ /dev/null @@ -1,154 +0,0 @@ -import sys -import codecs -from unittest import TestCase - -import simplejson as json -from simplejson.compat import unichr, text_type, b, BytesIO - -class TestUnicode(TestCase): - def test_encoding1(self): - encoder = json.JSONEncoder(encoding='utf-8') - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - s = u.encode('utf-8') - ju = encoder.encode(u) - js = encoder.encode(s) - self.assertEqual(ju, js) - - def test_encoding2(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - s = u.encode('utf-8') - ju = json.dumps(u, encoding='utf-8') - js = json.dumps(s, encoding='utf-8') - self.assertEqual(ju, js) - - def test_encoding3(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps(u) - self.assertEqual(j, '"\\u03b1\\u03a9"') - - def test_encoding4(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps([u]) - self.assertEqual(j, '["\\u03b1\\u03a9"]') - - def test_encoding5(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps(u, ensure_ascii=False) - self.assertEqual(j, u'"' + u + u'"') - - def test_encoding6(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps([u], ensure_ascii=False) - self.assertEqual(j, u'["' + u + u'"]') - - def test_big_unicode_encode(self): - u = u'\U0001d120' - self.assertEqual(json.dumps(u), '"\\ud834\\udd20"') - self.assertEqual(json.dumps(u, ensure_ascii=False), u'"\U0001d120"') - - def test_big_unicode_decode(self): - u = u'z\U0001d120x' - self.assertEqual(json.loads('"' + u + '"'), u) - self.assertEqual(json.loads('"z\\ud834\\udd20x"'), u) - - def test_unicode_decode(self): - for i in range(0, 0xd7ff): - u = unichr(i) - #s = '"\\u{0:04x}"'.format(i) - s = '"\\u%04x"' % (i,) - self.assertEqual(json.loads(s), u) - - def test_object_pairs_hook_with_unicode(self): - s = u'{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}' - p = [(u"xkd", 1), (u"kcw", 2), (u"art", 3), (u"hxm", 4), - (u"qrt", 5), (u"pad", 6), (u"hoy", 7)] - self.assertEqual(json.loads(s), eval(s)) - self.assertEqual(json.loads(s, object_pairs_hook=lambda x: x), p) - od = json.loads(s, object_pairs_hook=json.OrderedDict) - self.assertEqual(od, json.OrderedDict(p)) - self.assertEqual(type(od), json.OrderedDict) - # the object_pairs_hook takes priority over the object_hook - self.assertEqual(json.loads(s, - object_pairs_hook=json.OrderedDict, - object_hook=lambda x: None), - json.OrderedDict(p)) - - - def test_default_encoding(self): - self.assertEqual(json.loads(u'{"a": "\xe9"}'.encode('utf-8')), - {'a': u'\xe9'}) - - def test_unicode_preservation(self): - self.assertEqual(type(json.loads(u'""')), text_type) - self.assertEqual(type(json.loads(u'"a"')), text_type) - self.assertEqual(type(json.loads(u'["a"]')[0]), text_type) - - def test_ensure_ascii_false_returns_unicode(self): - # http://code.google.com/p/simplejson/issues/detail?id=48 - self.assertEqual(type(json.dumps([], ensure_ascii=False)), text_type) - self.assertEqual(type(json.dumps(0, ensure_ascii=False)), text_type) - self.assertEqual(type(json.dumps({}, ensure_ascii=False)), text_type) - self.assertEqual(type(json.dumps("", ensure_ascii=False)), text_type) - - def test_ensure_ascii_false_bytestring_encoding(self): - # http://code.google.com/p/simplejson/issues/detail?id=48 - doc1 = {u'quux': b('Arr\xc3\xaat sur images')} - doc2 = {u'quux': u'Arr\xeat sur images'} - doc_ascii = '{"quux": "Arr\\u00eat sur images"}' - doc_unicode = u'{"quux": "Arr\xeat sur images"}' - self.assertEqual(json.dumps(doc1), doc_ascii) - self.assertEqual(json.dumps(doc2), doc_ascii) - self.assertEqual(json.dumps(doc1, ensure_ascii=False), doc_unicode) - self.assertEqual(json.dumps(doc2, ensure_ascii=False), doc_unicode) - - def test_ensure_ascii_linebreak_encoding(self): - # http://timelessrepo.com/json-isnt-a-javascript-subset - s1 = u'\u2029\u2028' - s2 = s1.encode('utf8') - expect = '"\\u2029\\u2028"' - expect_non_ascii = u'"\u2029\u2028"' - self.assertEqual(json.dumps(s1), expect) - self.assertEqual(json.dumps(s2), expect) - self.assertEqual(json.dumps(s1, ensure_ascii=False), expect_non_ascii) - self.assertEqual(json.dumps(s2, ensure_ascii=False), expect_non_ascii) - - def test_invalid_escape_sequences(self): - # incomplete escape sequence - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u1') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u12') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u123') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u1234') - # invalid escape sequence - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u123x"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u12x4"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u1x34"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ux234"') - if sys.maxunicode > 65535: - # invalid escape sequence for low surrogate - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u0"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u00"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u000"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u000x"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u00x0"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u0x00"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\ux000"') - - def test_ensure_ascii_still_works(self): - # in the ascii range, ensure that everything is the same - for c in map(unichr, range(0, 127)): - self.assertEqual( - json.dumps(c, ensure_ascii=False), - json.dumps(c)) - snowman = u'\N{SNOWMAN}' - self.assertEqual( - json.dumps(c, ensure_ascii=False), - '"' + c + '"') - - def test_strip_bom(self): - content = u"\u3053\u3093\u306b\u3061\u308f" - json_doc = codecs.BOM_UTF8 + b(json.dumps(content)) - self.assertEqual(json.load(BytesIO(json_doc)), content) - for doc in json_doc, json_doc.decode('utf8'): - self.assertEqual(json.loads(doc), content) diff --git a/contrib/python/simplejson/py3/simplejson/tool.py b/contrib/python/simplejson/py3/simplejson/tool.py deleted file mode 100644 index 062e8e2c181..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tool.py +++ /dev/null @@ -1,42 +0,0 @@ -r"""Command-line tool to validate and pretty-print JSON - -Usage:: - - $ echo '{"json":"obj"}' | python -m simplejson.tool - { - "json": "obj" - } - $ echo '{ 1.2:3.4}' | python -m simplejson.tool - Expecting property name: line 1 column 2 (char 2) - -""" -from __future__ import with_statement -import sys -import simplejson as json - -def main(): - if len(sys.argv) == 1: - infile = sys.stdin - outfile = sys.stdout - elif len(sys.argv) == 2: - infile = open(sys.argv[1], 'r') - outfile = sys.stdout - elif len(sys.argv) == 3: - infile = open(sys.argv[1], 'r') - outfile = open(sys.argv[2], 'w') - else: - raise SystemExit(sys.argv[0] + " [infile [outfile]]") - with infile: - try: - obj = json.load(infile, - object_pairs_hook=json.OrderedDict, - use_decimal=True) - except ValueError: - raise SystemExit(sys.exc_info()[1]) - with outfile: - json.dump(obj, outfile, sort_keys=True, indent=' ', use_decimal=True) - outfile.write('\n') - - -if __name__ == '__main__': - main() diff --git a/contrib/python/simplejson/py3/tests/ya.make b/contrib/python/simplejson/py3/tests/ya.make deleted file mode 100644 index 3f788396a7c..00000000000 --- a/contrib/python/simplejson/py3/tests/ya.make +++ /dev/null @@ -1,47 +0,0 @@ -PY3TEST() - -PEERDIR( - contrib/python/simplejson -) - -SRCDIR( - contrib/python/simplejson/py3/simplejson/tests -) - -TEST_SRCS( - __init__.py - test_bigint_as_string.py - test_bitsize_int_as_string.py - test_check_circular.py - test_decimal.py - test_decode.py - test_default.py - test_dump.py - test_encode_basestring_ascii.py - test_encode_for_html.py - test_errors.py - test_fail.py - test_float.py - test_for_json.py - test_indent.py - test_item_sort_key.py - test_iterable.py - test_namedtuple.py - test_pass1.py - test_pass2.py - test_pass3.py - test_raw_json.py - test_recursion.py - test_scanstring.py - test_separators.py - test_speedups.py - test_str_subclass.py - test_subclass.py - # test_tool.py - test_tuple.py - test_unicode.py -) - -NO_LINT() - -END() diff --git a/contrib/python/simplejson/py3/ya.make b/contrib/python/simplejson/py3/ya.make deleted file mode 100644 index 85e677db4bd..00000000000 --- a/contrib/python/simplejson/py3/ya.make +++ /dev/null @@ -1,48 +0,0 @@ -# Generated by devtools/yamaker (pypi). - -PY3_LIBRARY() - -VERSION(3.20.2) - -LICENSE(MIT) - -NO_COMPILER_WARNINGS() - -NO_LINT() - -NO_CHECK_IMPORTS( - simplejson.ordered_dict -) - -SRCS( - simplejson/_speedups.c -) - -PY_REGISTER( - simplejson._speedups -) - -PY_SRCS( - TOP_LEVEL - simplejson/__init__.py - simplejson/compat.py - simplejson/decoder.py - simplejson/encoder.py - simplejson/errors.py - simplejson/ordered_dict.py - simplejson/raw_json.py - simplejson/scanner.py - simplejson/tool.py -) - -RESOURCE_FILES( - PREFIX contrib/python/simplejson/py3/ - .dist-info/METADATA - .dist-info/top_level.txt -) - -END() - -RECURSE_FOR_TESTS( - tests -) diff --git a/contrib/python/simplejson/ya.make b/contrib/python/simplejson/ya.make deleted file mode 100644 index 8ba230decd8..00000000000 --- a/contrib/python/simplejson/ya.make +++ /dev/null @@ -1,20 +0,0 @@ -PY23_LIBRARY() - -LICENSE(Service-Py23-Proxy) - -VERSION(Service-proxy-version) - -IF (PYTHON2) - PEERDIR(contrib/python/simplejson/py2) -ELSE() - PEERDIR(contrib/python/simplejson/py3) -ENDIF() - -NO_LINT() - -END() - -RECURSE( - py2 - py3 -) diff --git a/contrib/python/textual/.dist-info/METADATA b/contrib/python/textual/.dist-info/METADATA index f3bb97ae79a..bdcd6666e66 100644 --- a/contrib/python/textual/.dist-info/METADATA +++ b/contrib/python/textual/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.4 Name: textual -Version: 8.2.7 +Version: 8.2.8 Summary: Modern Text User Interface framework License: MIT License-File: LICENSE diff --git a/contrib/python/textual/textual/_compositor.py b/contrib/python/textual/textual/_compositor.py index ee4c231f240..9af38a23c7e 100644 --- a/contrib/python/textual/textual/_compositor.py +++ b/contrib/python/textual/textual/_compositor.py @@ -923,11 +923,8 @@ class Compositor: x -= region.x + gutter_left y -= region.y + gutter_right - if y < 0: - return None, None - - if x < 0: - return widget, Offset(0, y) + if x < 0 or y < 0: + return widget, Offset(max(0, x), max(0, y)) visible_screen_stack.set(widget.app._background_screens) line = widget.render_line(y) diff --git a/contrib/python/textual/textual/_xterm_parser.py b/contrib/python/textual/textual/_xterm_parser.py index eafa7414ecb..df9112370cd 100644 --- a/contrib/python/textual/textual/_xterm_parser.py +++ b/contrib/python/textual/textual/_xterm_parser.py @@ -39,7 +39,7 @@ SPECIAL_SEQUENCES = {BRACKETED_PASTE_START, BRACKETED_PASTE_END, FOCUSIN, FOCUSO """Set of special sequences.""" _re_extended_key: Final[re.Pattern[str]] = re.compile( - r"\x1b\[((?:\d*;?){2,3})([u~ABCDEFHPQRS])" + r"\x1b\[((?:[\d:]*;?){2,3})([u~ABCDEFHPQRS])" ) _re_in_band_window_resize: Final[re.Pattern[str]] = re.compile( r"\x1b\[48;(\d+(?:\:.*?)?);(\d+(?:\:.*?)?);(\d+(?:\:.*?)?);(\d+(?:\:.*?)?)t" @@ -334,8 +334,26 @@ class XTermParser(Parser[Message]): self._debug_log_file.close() self._debug_log_file = None + @classmethod + def _parse_colon_codepoints(cls, text_str: str) -> list[str | None]: + """Convert codepoints split on colons in to a list of characters. + + Args: + text_str: String with groups of digits, separated by one or more colons. + + Returns: + A list of characters. + """ + if not text_str: + return [None] + characters: list[str | None] = [ + chr(int(part)) if part.isdecimal() else chr(1) + for part in text_str.split(":") + ] + return characters + @lru_cache(maxsize=1024) - def _parse_extended_key(self, sequence: str) -> events.Key | None: + def _parse_extended_key(self, sequence: str) -> list[events.Key] | None: """Parse a Kitty sequence. Args: @@ -348,38 +366,47 @@ class XTermParser(Parser[Message]): if (match := _re_extended_key.fullmatch(sequence)) is None: return None + key_events: list[events.Key] = [] + codes, end = match.groups(default="") codepoint_str, modifiers_str, text_str, *_ = codes.split(";") + ["", "", ""] - codepoint = int(codepoint_str or "1") modifiers = int(modifiers_str or "0") - text = chr(int(text_str)) if text_str else None - if not (key := FUNCTIONAL_KEYS.get(f"{codepoint}{end}", "")): - key = _character_to_key(text if text else chr(codepoint)) + for text in self._parse_colon_codepoints(text_str): + if not (key := FUNCTIONAL_KEYS.get(f"{codepoint}{end}", "")): + key = _character_to_key(text if text else chr(codepoint)) - key_tokens: list[str] = [] - # The modifier is redundant on a modifier key - if modifiers and key not in MODIFIER_FUNCTIONAL_KEYS and text_str is not None: - modifier_bits = int(modifiers) - 1 - # Not convinced of the utility in reporting caps_lock and num_lock - MODIFIERS = ("alt", "ctrl", "super", "hyper", "meta") - # Ignore caps_lock and num_lock modifiers - if modifier_bits & 1 and (text is None or text.isspace()): - key_tokens.append("shift") - for bit, modifier in enumerate(MODIFIERS, 1): - if modifier == "alt" and text is not None: - continue - if modifier_bits & (1 << bit): - key_tokens.append(modifier) + key_tokens: list[str] = [] + # The modifier is redundant on a modifier key + if ( + modifiers + and key not in MODIFIER_FUNCTIONAL_KEYS + and text_str is not None + ): + modifier_bits = int(modifiers) - 1 + # Not convinced of the utility in reporting caps_lock and num_lock + MODIFIERS = ("alt", "ctrl", "super", "hyper", "meta") + # Ignore caps_lock and num_lock modifiers + if modifier_bits & 1 and (text is None or text.isspace()): + key_tokens.append("shift") + for bit, modifier in enumerate(MODIFIERS, 1): + if modifier == "alt" and text is not None: + continue + if modifier_bits & (1 << bit): + key_tokens.append(modifier) - key_tokens.sort() - if key is not None: - key_tokens.append(key) - return events.Key( - "+".join(key_tokens), - text or (None if modifiers else SPECIAL_KEY_TO_CHARACTER.get(key, None)), - ) + key_tokens.sort() + if key is not None: + key_tokens.append(key) + key_events.append( + events.Key( + "+".join(key_tokens), + text + or (None if modifiers else SPECIAL_KEY_TO_CHARACTER.get(key, None)), + ) + ) + return key_events def _sequence_to_key_events( self, sequence: str, alt: bool = False @@ -395,9 +422,10 @@ class XTermParser(Parser[Message]): if ( not constants.DISABLE_KITTY_KEY - and (key := self._parse_extended_key(sequence)) is not None + and (keys := self._parse_extended_key(sequence)) is not None ): - yield key.copy() + for key in keys: + yield key.copy() return keys = ANSI_SEQUENCES_KEYS.get(sequence) diff --git a/contrib/python/textual/textual/pilot.py b/contrib/python/textual/textual/pilot.py index c387b258276..76adae9d594 100644 --- a/contrib/python/textual/textual/pilot.py +++ b/contrib/python/textual/textual/pilot.py @@ -438,7 +438,7 @@ class Pilot(Generic[ReturnType]): ) offset = Offset(message_arguments["x"], message_arguments["y"]) - if offset not in screen.region: + if offset not in screen.size.region: raise OutOfBounds( "Target offset is outside of currently-visible screen region." ) diff --git a/contrib/python/textual/textual/screen.py b/contrib/python/textual/textual/screen.py index af9070538e7..c8b1d1462d4 100644 --- a/contrib/python/textual/textual/screen.py +++ b/contrib/python/textual/textual/screen.py @@ -1838,8 +1838,11 @@ class Screen(Generic[ScreenResultType], Widget): if select_offset is not None: content_widget = select_widget content_offset = select_offset - assert isinstance(content_widget.parent, Widget) - container = content_widget.parent + container = ( + content_widget + if isinstance(content_widget, Screen) + else content_widget.parent + ) else: content_widget = None container = select_widget @@ -1884,6 +1887,7 @@ class Screen(Generic[ScreenResultType], Widget): select_widget, select_offset = self.get_widget_and_offset_at( event.x, event.y ) + if ( select_widget is not None and select_widget.allow_select @@ -1893,8 +1897,11 @@ class Screen(Generic[ScreenResultType], Widget): if select_offset is not None: content_widget = select_widget content_offset = select_offset - assert isinstance(content_widget.parent, Widget) - container = content_widget.parent + container = ( + content_widget + if isinstance(content_widget, Screen) + else content_widget.parent + ) else: content_widget = None container = select_widget diff --git a/contrib/python/textual/textual/widgets/_input.py b/contrib/python/textual/textual/widgets/_input.py index 3aa2a945bdb..bcf2c2994de 100644 --- a/contrib/python/textual/textual/widgets/_input.py +++ b/contrib/python/textual/textual/widgets/_input.py @@ -122,9 +122,14 @@ class Input(ScrollView): Binding( "ctrl+w", "delete_left_word", "Delete left to start of word", show=False ), - Binding("ctrl+u", "delete_left_all", "Delete all to the left", show=False), Binding( - "ctrl+backspace", + "ctrl+u,super+backspace", + "delete_left_all", + "Delete all to the left", + show=False, + ), + Binding( + "ctrl+backspace,alt+backspace", "delete_right_word", "Delete right to start of word", show=False, @@ -145,6 +150,7 @@ class Input(ScrollView): | shift+right | Move cursor right and select. | | ctrl+right | Move the cursor one word to the right. | | backspace | Delete the character to the left of the cursor. | + | ctrl+backspace,alt+backspace | Delete right to start of word. | | ctrl+shift+right | Move cursor right a word and select. | | ctrl+shift+a | Select all text in the input. | | home,ctrl+a | Go to the beginning of the input. | @@ -154,7 +160,7 @@ class Input(ScrollView): | delete,ctrl+d | Delete the character to the right of the cursor. | | enter | Submit the current value of the input. | | ctrl+w | Delete the word to the left of the cursor. | - | ctrl+u | Delete everything to the left of the cursor. | + | ctrl+u,super+backspace | Delete everything to the left of the cursor. | | ctrl+f | Delete the word to the right of the cursor. | | ctrl+k | Delete everything to the right of the cursor. | | ctrl+x | Cut selected text. | diff --git a/contrib/python/textual/textual/widgets/_text_area.py b/contrib/python/textual/textual/widgets/_text_area.py index d970a31c5a7..1ca74675e54 100644 --- a/contrib/python/textual/textual/widgets/_text_area.py +++ b/contrib/python/textual/textual/widgets/_text_area.py @@ -354,7 +354,7 @@ TextArea { show=False, ), Binding( - "ctrl+w,ctrl+backspace", + "ctrl+w,ctrl+backspace,alt+backspace", "delete_word_left", "Delete left to start of word", show=False, @@ -390,7 +390,7 @@ TextArea { show=False, ), Binding( - "ctrl+u", + "ctrl+u,super+backspace", "delete_to_start_of_line", "Delete to line start", show=False, @@ -446,7 +446,7 @@ TextArea { | delete,ctrl+d | Delete character to the right of cursor. | | alt+delete | Delete from cursor to end of the word. | | ctrl+shift+k | Delete the current line. | - | ctrl+u | Delete from cursor to the start of the line. | + | ctrl+u,super+backspace | Delete from cursor to the start of the line. | | ctrl+k | Delete from cursor to the end of the line. | | f6 | Select the current line. | | f7 | Select all text in the document. | diff --git a/contrib/python/textual/ya.make b/contrib/python/textual/ya.make index 84a9205d17e..fd182d0171f 100644 --- a/contrib/python/textual/ya.make +++ b/contrib/python/textual/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(8.2.7) +VERSION(8.2.8) LICENSE(MIT) diff --git a/contrib/python/toml/py3/ya.make b/contrib/python/toml/py3/ya.make index be97015fd78..9d29cc702f1 100644 --- a/contrib/python/toml/py3/ya.make +++ b/contrib/python/toml/py3/ya.make @@ -2,6 +2,8 @@ PY3_LIBRARY() +SUBSCRIBER(g:python-contrib) + VERSION(0.10.2) LICENSE(MIT) |