diff options
| author | uzhas <[email protected]> | 2026-07-17 16:30:05 +0300 |
|---|---|---|
| committer | uzhas <[email protected]> | 2026-07-17 19:17:43 +0300 |
| commit | 09cd02b72802d18ce7257ba17ef8337e669be85c (patch) | |
| tree | fdd42e73e6be297d7bf86399627dc56b118ec102 | |
| parent | b1cc4143fea8621e13dedcb57a09ae920f917a94 (diff) | |
move dqrun to yt/
- Move `contrib/ydb/library/yql/providers/dq/{config,local_gateway,service,stats_collector}` and gateway/control implementation to `yt/yql/providers/dq/`
- Move `contrib/ydb/library/yql/tools/dqrun` to `yt/yql/tools/dqrun`
- Clean up `contrib/ydb` provider (slim `yql_dq_gateway.h`, PEERDIR updates) and fix consumers (`scheduler`, `global_worker_manager`, `pq/provider/ut`, benchmarks, solomon tests)
## Test plan
- [x] `./ya make --build debug contrib/ydb/library/yql/providers/dq/provider contrib/ydb/library/yql/providers/dq/provider/exec yt/yql/providers/dq/gateway yt/yql/providers/dq/local_gateway yt/yql/providers/dq/service yt/yql/tools/dqrun`
- [x] `./ya make --build debug -tA yt/yql/providers/dq/provider/ut`
commit_hash:18a321f318f5903167749b88541a79b34ab1c8d7
131 files changed, 16 insertions, 20167 deletions
diff --git a/contrib/deprecated/python/enum34/tests/ya.make b/contrib/deprecated/python/enum34/tests/ya.make index f1b8300430c..6825b19a484 100644 --- a/contrib/deprecated/python/enum34/tests/ya.make +++ b/contrib/deprecated/python/enum34/tests/ya.make @@ -1,5 +1,7 @@ PY2TEST() +SUBSCRIBER(g:python-contrib) + SRCDIR(contrib/deprecated/python/enum34/enum) TEST_SRCS( diff --git a/contrib/python/chardet/py2/tests/ya.make b/contrib/python/chardet/py2/tests/ya.make index 3795bfa7fbd..b5e332be138 100644 --- a/contrib/python/chardet/py2/tests/ya.make +++ b/contrib/python/chardet/py2/tests/ya.make @@ -1,5 +1,7 @@ PY2TEST() +SUBSCRIBER(g:python-contrib ) + SRCDIR(contrib/python/chardet/py2) TEST_SRCS( diff --git a/contrib/python/chardet/py3/tests/ya.make b/contrib/python/chardet/py3/tests/ya.make index c1f16c7df7f..16b148c70e9 100644 --- a/contrib/python/chardet/py3/tests/ya.make +++ b/contrib/python/chardet/py3/tests/ya.make @@ -1,5 +1,7 @@ PY3TEST() +SUBSCRIBER(g:python-contrib ) + SRCDIR(contrib/python/chardet/py3) TEST_SRCS( diff --git a/contrib/python/chardet/py3/ya.make b/contrib/python/chardet/py3/ya.make index b14449360dd..14f394da7c6 100644 --- a/contrib/python/chardet/py3/ya.make +++ b/contrib/python/chardet/py3/ya.make @@ -2,6 +2,8 @@ PY3_LIBRARY() +SUBSCRIBER(g:python-contrib) + VERSION(5.2.0) LICENSE(LGPL-2.1-or-later) diff --git a/contrib/python/ipython-genutils/py3/ya.make b/contrib/python/ipython-genutils/py3/ya.make index df370592060..373400ffb8b 100644 --- a/contrib/python/ipython-genutils/py3/ya.make +++ b/contrib/python/ipython-genutils/py3/ya.make @@ -2,6 +2,8 @@ PY3_LIBRARY() +SUBSCRIBER(g:python-contrib) + VERSION(0.2.0) LICENSE(BSD-3-Clause) diff --git a/contrib/python/pickleshare/py3/ya.make b/contrib/python/pickleshare/py3/ya.make index 84629e7786f..cdc377b5045 100644 --- a/contrib/python/pickleshare/py3/ya.make +++ b/contrib/python/pickleshare/py3/ya.make @@ -2,6 +2,8 @@ PY3_LIBRARY() +SUBSCRIBER(g:python-contrib) + VERSION(0.7.5) LICENSE(MIT) diff --git a/contrib/python/simplegeneric/py3/ya.make b/contrib/python/simplegeneric/py3/ya.make index 65e5d02fa45..fe07c8bd96a 100644 --- a/contrib/python/simplegeneric/py3/ya.make +++ b/contrib/python/simplegeneric/py3/ya.make @@ -2,6 +2,8 @@ PY3_LIBRARY() +SUBSCRIBER(g:python-contrib) + VERSION(0.8.1) LICENSE(ZPL-2.1) diff --git a/contrib/python/simplejson/py2/.dist-info/METADATA b/contrib/python/simplejson/py2/.dist-info/METADATA deleted file mode 100644 index e95da1917c6..00000000000 --- a/contrib/python/simplejson/py2/.dist-info/METADATA +++ /dev/null @@ -1,71 +0,0 @@ -Metadata-Version: 2.1 -Name: simplejson -Version: 3.20.2 -Summary: Simple, fast, extensible JSON encoder/decoder for Python -Home-page: https://github.com/simplejson/simplejson -Author: Bob Ippolito -Author-email: [email protected] -License: MIT License -Platform: any -Classifier: Development Status :: 5 - Production/Stable -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: MIT License -Classifier: License :: OSI Approved :: Academic Free License (AFL) -Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 2 -Classifier: Programming Language :: Python :: 2.5 -Classifier: Programming Language :: Python :: 2.6 -Classifier: Programming Language :: Python :: 2.7 -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.3 -Classifier: Programming Language :: Python :: 3.4 -Classifier: Programming Language :: Python :: 3.5 -Classifier: Programming Language :: Python :: 3.6 -Classifier: Programming Language :: Python :: 3.7 -Classifier: Programming Language :: Python :: 3.8 -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Programming Language :: Python :: Implementation :: CPython -Classifier: Programming Language :: Python :: Implementation :: PyPy -Classifier: Topic :: Software Development :: Libraries :: Python Modules -Requires-Python: >=2.5, !=3.0.*, !=3.1.*, !=3.2.* - -simplejson ----------- - -simplejson is a simple, fast, complete, correct and extensible -JSON <http://json.org> encoder and decoder for Python 3.3+ -with legacy support for Python 2.5+. It is pure Python code -with no dependencies, but includes an optional C extension -for a serious speed boost. - -The latest documentation for simplejson can be read online here: -https://simplejson.readthedocs.io/ - -simplejson is the externally maintained development version of the -json library included with Python (since 2.6). This version is tested -with the latest Python 3.8 and maintains backwards compatibility -with Python 3.3+ and the legacy Python 2.5 - Python 2.7 releases. - -The encoder can be specialized to provide serialization in any kind of -situation, without any special support by the objects to be serialized -(somewhat like pickle). This is best done with the ``default`` kwarg -to dumps. - -The decoder can handle incoming JSON strings of any specified encoding -(UTF-8 by default). It can also be specialized to post-process JSON -objects with the ``object_hook`` or ``object_pairs_hook`` kwargs. This -is particularly useful for implementing protocols such as JSON-RPC -that have a richer type system than JSON itself. - -For those of you that have legacy systems to maintain, there is a -very old fork of simplejson in the `python2.2`_ branch that supports -Python 2.2. This is based on a very old version of simplejson, -is not maintained, and should only be used as a last resort. - -.. _python2.2: https://github.com/simplejson/simplejson/tree/python2.2 - - diff --git a/contrib/python/simplejson/py2/.dist-info/top_level.txt b/contrib/python/simplejson/py2/.dist-info/top_level.txt deleted file mode 100644 index 322630ee75f..00000000000 --- a/contrib/python/simplejson/py2/.dist-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -simplejson diff --git a/contrib/python/simplejson/py2/LICENSE.txt b/contrib/python/simplejson/py2/LICENSE.txt deleted file mode 100644 index e05f49c3fd0..00000000000 --- a/contrib/python/simplejson/py2/LICENSE.txt +++ /dev/null @@ -1,79 +0,0 @@ -simplejson is dual-licensed software. It is available under the terms -of the MIT license, or the Academic Free License version 2.1. The full -text of each license agreement is included below. This code is also -licensed to the Python Software Foundation (PSF) under a Contributor -Agreement. - -MIT License -=========== - -Copyright (c) 2006 Bob Ippolito - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -Academic Free License v. 2.1 -============================ - -Copyright (c) 2006 Bob Ippolito. All rights reserved. - -This Academic Free License (the "License") applies to any original work of authorship (the "Original Work") whose owner (the "Licensor") has placed the following notice immediately following the copyright notice for the Original Work: - -Licensed under the Academic Free License version 2.1 - -1) Grant of Copyright License. Licensor hereby grants You a world-wide, royalty-free, non-exclusive, perpetual, sublicenseable license to do the following: - -a) to reproduce the Original Work in copies; - -b) to prepare derivative works ("Derivative Works") based upon the Original Work; - -c) to distribute copies of the Original Work and Derivative Works to the public; - -d) to perform the Original Work publicly; and - -e) to display the Original Work publicly. - -2) Grant of Patent License. Licensor hereby grants You a world-wide, royalty-free, non-exclusive, perpetual, sublicenseable license, under patent claims owned or controlled by the Licensor that are embodied in the Original Work as furnished by the Licensor, to make, use, sell and offer for sale the Original Work and Derivative Works. - -3) Grant of Source Code License. The term "Source Code" means the preferred form of the Original Work for making modifications to it and all available documentation describing how to modify the Original Work. Licensor hereby agrees to provide a machine-readable copy of the Source Code of the Original Work along with each copy of the Original Work that Licensor distributes. Licensor reserves the right to satisfy this obligation by placing a machine-readable copy of the Source Code in an information repository reasonably calculated to permit inexpensive and convenient access by You for as long as Licensor continues to distribute the Original Work, and by publishing the address of that information repository in a notice immediately following the copyright notice that applies to the Original Work. - -4) Exclusions From License Grant. Neither the names of Licensor, nor the names of any contributors to the Original Work, nor any of their trademarks or service marks, may be used to endorse or promote products derived from this Original Work without express prior written permission of the Licensor. Nothing in this License shall be deemed to grant any rights to trademarks, copyrights, patents, trade secrets or any other intellectual property of Licensor except as expressly stated herein. No patent license is granted to make, use, sell or offer to sell embodiments of any patent claims other than the licensed claims defined in Section 2. No right is granted to the trademarks of Licensor even if such marks are included in the Original Work. Nothing in this License shall be interpreted to prohibit Licensor from licensing under different terms from this License any Original Work that Licensor otherwise would have a right to license. - -5) This section intentionally omitted. - -6) Attribution Rights. You must retain, in the Source Code of any Derivative Works that You create, all copyright, patent or trademark notices from the Source Code of the Original Work, as well as any notices of licensing and any descriptive text identified therein as an "Attribution Notice." You must cause the Source Code for any Derivative Works that You create to carry a prominent Attribution Notice reasonably calculated to inform recipients that You have modified the Original Work. - -7) Warranty of Provenance and Disclaimer of Warranty. Licensor warrants that the copyright in and to the Original Work and the patent rights granted herein by Licensor are owned by the Licensor or are sublicensed to You under the terms of this License with the permission of the contributor(s) of those copyrights and patent rights. Except as expressly stated in the immediately proceeding sentence, the Original Work is provided under this License on an "AS IS" BASIS and WITHOUT WARRANTY, either express or implied, including, without limitation, the warranties of NON-INFRINGEMENT, MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY OF THE ORIGINAL WORK IS WITH YOU. This DISCLAIMER OF WARRANTY constitutes an essential part of this License. No license to Original Work is granted hereunder except under this disclaimer. - -8) Limitation of Liability. Under no circumstances and under no legal theory, whether in tort (including negligence), contract, or otherwise, shall the Licensor be liable to any person for any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or the use of the Original Work including, without limitation, damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses. This limitation of liability shall not apply to liability for death or personal injury resulting from Licensor's negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. - -9) Acceptance and Termination. If You distribute copies of the Original Work or a Derivative Work, You must make a reasonable effort under the circumstances to obtain the express assent of recipients to the terms of this License. Nothing else but this License (or another written agreement between Licensor and You) grants You permission to create Derivative Works based upon the Original Work or to exercise any of the rights granted in Section 1 herein, and any attempt to do so except under the terms of this License (or another written agreement between Licensor and You) is expressly prohibited by U.S. copyright law, the equivalent laws of other countries, and by international treaty. Therefore, by exercising any of the rights granted to You in Section 1 herein, You indicate Your acceptance of this License and all of its terms and conditions. - -10) Termination for Patent Action. This License shall terminate automatically and You may no longer exercise any of the rights granted to You by this License as of the date You commence an action, including a cross-claim or counterclaim, against Licensor or any licensee alleging that the Original Work infringes a patent. This termination provision shall not apply for an action alleging patent infringement by combinations of the Original Work with other software or hardware. - -11) Jurisdiction, Venue and Governing Law. Any action or suit relating to this License may be brought only in the courts of a jurisdiction wherein the Licensor resides or in which Licensor conducts its primary business, and under the laws of that jurisdiction excluding its conflict-of-law provisions. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any use of the Original Work outside the scope of this License or after its termination shall be subject to the requirements and penalties of the U.S. Copyright Act, 17 U.S.C. § 101 et seq., the equivalent laws of other countries, and international treaty. This section shall survive the termination of this License. - -12) Attorneys Fees. In any action to enforce the terms of this License or seeking damages relating thereto, the prevailing party shall be entitled to recover its costs and expenses, including, without limitation, reasonable attorneys' fees and costs incurred in connection with such action, including any appeal of such action. This section shall survive the termination of this License. - -13) Miscellaneous. This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. - -14) Definition of "You" in This License. "You" throughout this License, whether in upper or lower case, means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License. For legal entities, "You" includes any entity that controls, is controlled by, or is under common control with you. For purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. - -15) Right to Use. You may use the Original Work in all ways not otherwise restricted or conditioned by this License or by law, and Licensor promises not to interfere with or be responsible for such uses by You. - -This license is Copyright (C) 2003-2004 Lawrence E. Rosen. All rights reserved. Permission is hereby granted to copy and distribute this license without modification. This license may not be modified without the express written permission of its copyright owner. diff --git a/contrib/python/simplejson/py2/README.rst b/contrib/python/simplejson/py2/README.rst deleted file mode 100644 index 6580ddac899..00000000000 --- a/contrib/python/simplejson/py2/README.rst +++ /dev/null @@ -1,34 +0,0 @@ -simplejson ----------- - -simplejson is a simple, fast, complete, correct and extensible -JSON <http://json.org> encoder and decoder for Python 3.3+ -with legacy support for Python 2.5+. It is pure Python code -with no dependencies, but includes an optional C extension -for a serious speed boost. - -The latest documentation for simplejson can be read online here: -https://simplejson.readthedocs.io/ - -simplejson is the externally maintained development version of the -json library included with Python (since 2.6). This version is tested -with the latest Python 3.8 and maintains backwards compatibility -with Python 3.3+ and the legacy Python 2.5 - Python 2.7 releases. - -The encoder can be specialized to provide serialization in any kind of -situation, without any special support by the objects to be serialized -(somewhat like pickle). This is best done with the ``default`` kwarg -to dumps. - -The decoder can handle incoming JSON strings of any specified encoding -(UTF-8 by default). It can also be specialized to post-process JSON -objects with the ``object_hook`` or ``object_pairs_hook`` kwargs. This -is particularly useful for implementing protocols such as JSON-RPC -that have a richer type system than JSON itself. - -For those of you that have legacy systems to maintain, there is a -very old fork of simplejson in the `python2.2`_ branch that supports -Python 2.2. This is based on a very old version of simplejson, -is not maintained, and should only be used as a last resort. - -.. _python2.2: https://github.com/simplejson/simplejson/tree/python2.2 diff --git a/contrib/python/simplejson/py2/simplejson/__init__.py b/contrib/python/simplejson/py2/simplejson/__init__.py deleted file mode 100644 index a526d8c7823..00000000000 --- a/contrib/python/simplejson/py2/simplejson/__init__.py +++ /dev/null @@ -1,562 +0,0 @@ -r"""JSON (JavaScript Object Notation) <http://json.org> is a subset of -JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data -interchange format. - -:mod:`simplejson` exposes an API familiar to users of the standard library -:mod:`marshal` and :mod:`pickle` modules. It is the externally maintained -version of the :mod:`json` library contained in Python 2.6, but maintains -compatibility back to Python 2.5 and (currently) has significant performance -advantages, even without using the optional C extension for speedups. - -Encoding basic Python object hierarchies:: - - >>> import simplejson as json - >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}]) - '["foo", {"bar": ["baz", null, 1.0, 2]}]' - >>> print(json.dumps("\"foo\bar")) - "\"foo\bar" - >>> print(json.dumps(u'\u1234')) - "\u1234" - >>> print(json.dumps('\\')) - "\\" - >>> print(json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True)) - {"a": 0, "b": 0, "c": 0} - >>> from simplejson.compat import StringIO - >>> io = StringIO() - >>> json.dump(['streaming API'], io) - >>> io.getvalue() - '["streaming API"]' - -Compact encoding:: - - >>> import simplejson as json - >>> obj = [1,2,3,{'4': 5, '6': 7}] - >>> json.dumps(obj, separators=(',',':'), sort_keys=True) - '[1,2,3,{"4":5,"6":7}]' - -Pretty printing:: - - >>> import simplejson as json - >>> print(json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=' ')) - { - "4": 5, - "6": 7 - } - -Decoding JSON:: - - >>> import simplejson as json - >>> obj = [u'foo', {u'bar': [u'baz', None, 1.0, 2]}] - >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj - True - >>> json.loads('"\\"foo\\bar"') == u'"foo\x08ar' - True - >>> from simplejson.compat import StringIO - >>> io = StringIO('["streaming API"]') - >>> json.load(io)[0] == 'streaming API' - True - -Specializing JSON object decoding:: - - >>> import simplejson as json - >>> def as_complex(dct): - ... if '__complex__' in dct: - ... return complex(dct['real'], dct['imag']) - ... return dct - ... - >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}', - ... object_hook=as_complex) - (1+2j) - >>> from decimal import Decimal - >>> json.loads('1.1', parse_float=Decimal) == Decimal('1.1') - True - -Specializing JSON object encoding:: - - >>> import simplejson as json - >>> def encode_complex(obj): - ... if isinstance(obj, complex): - ... return [obj.real, obj.imag] - ... raise TypeError('Object of type %s is not JSON serializable' % - ... obj.__class__.__name__) - ... - >>> json.dumps(2 + 1j, default=encode_complex) - '[2.0, 1.0]' - >>> json.JSONEncoder(default=encode_complex).encode(2 + 1j) - '[2.0, 1.0]' - >>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j)) - '[2.0, 1.0]' - -Using simplejson.tool from the shell to validate and pretty-print:: - - $ echo '{"json":"obj"}' | python -m simplejson.tool - { - "json": "obj" - } - $ echo '{ 1.2:3.4}' | python -m simplejson.tool - Expecting property name: line 1 column 3 (char 2) - -Parsing multiple documents serialized as JSON lines (newline-delimited JSON):: - - >>> import simplejson as json - >>> def loads_lines(docs): - ... for doc in docs.splitlines(): - ... yield json.loads(doc) - ... - >>> sum(doc["count"] for doc in loads_lines('{"count":1}\n{"count":2}\n{"count":3}\n')) - 6 - -Serializing multiple objects to JSON lines (newline-delimited JSON):: - - >>> import simplejson as json - >>> def dumps_lines(objs): - ... for obj in objs: - ... yield json.dumps(obj, separators=(',',':')) + '\n' - ... - >>> ''.join(dumps_lines([{'count': 1}, {'count': 2}, {'count': 3}])) - '{"count":1}\n{"count":2}\n{"count":3}\n' - -""" -from __future__ import absolute_import -__version__ = '3.20.2' -__all__ = [ - 'dump', 'dumps', 'load', 'loads', - 'JSONDecoder', 'JSONDecodeError', 'JSONEncoder', - 'OrderedDict', 'simple_first', 'RawJSON' -] - -__author__ = 'Bob Ippolito <[email protected]>' - -from decimal import Decimal - -from .errors import JSONDecodeError -from .raw_json import RawJSON -from .decoder import JSONDecoder -from .encoder import JSONEncoder, JSONEncoderForHTML -def _import_OrderedDict(): - import collections - try: - return collections.OrderedDict - except AttributeError: - from . import ordered_dict - return ordered_dict.OrderedDict -OrderedDict = _import_OrderedDict() - -def _import_c_make_encoder(): - try: - from ._speedups import make_encoder - return make_encoder - except ImportError: - return None - -_default_encoder = JSONEncoder() - -def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, - allow_nan=False, cls=None, indent=None, separators=None, - encoding='utf-8', default=None, use_decimal=True, - namedtuple_as_object=True, tuple_as_array=True, - bigint_as_string=False, sort_keys=False, item_sort_key=None, - for_json=False, ignore_nan=False, int_as_string_bitcount=None, - iterable_as_array=False, **kw): - """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a - ``.write()``-supporting file-like object). - - If *skipkeys* is true then ``dict`` keys that are not basic types - (``str``, ``int``, ``long``, ``float``, ``bool``, ``None``) - will be skipped instead of raising a ``TypeError``. - - If *ensure_ascii* is false (default: ``True``), then the output may - contain non-ASCII characters, so long as they do not need to be escaped - by JSON. When it is true, all non-ASCII characters are escaped. - - If *allow_nan* is true (default: ``False``), then out of range ``float`` - values (``nan``, ``inf``, ``-inf``) will be serialized to - their JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``) - instead of raising a ValueError. See - *ignore_nan* for ECMA-262 compliant behavior. - - If *indent* is a string, then JSON array elements and object members - will be pretty-printed with a newline followed by that string repeated - for each level of nesting. ``None`` (the default) selects the most compact - representation without any newlines. - - If specified, *separators* should be an - ``(item_separator, key_separator)`` tuple. The default is ``(', ', ': ')`` - if *indent* is ``None`` and ``(',', ': ')`` otherwise. To get the most - compact JSON representation, you should specify ``(',', ':')`` to eliminate - whitespace. - - *encoding* is the character encoding for str instances, default is UTF-8. - - *default(obj)* is a function that should return a serializable version - of obj or raise ``TypeError``. The default simply raises ``TypeError``. - - If *use_decimal* is true (default: ``True``) then decimal.Decimal - will be natively serialized to JSON with full precision. - - If *namedtuple_as_object* is true (default: ``True``), - :class:`tuple` subclasses with ``_asdict()`` methods will be encoded - as JSON objects. - - If *tuple_as_array* is true (default: ``True``), - :class:`tuple` (and subclasses) will be encoded as JSON arrays. - - If *iterable_as_array* is true (default: ``False``), - any object not in the above table that implements ``__iter__()`` - will be encoded as a JSON array. - - If *bigint_as_string* is true (default: ``False``), ints 2**53 and higher - or lower than -2**53 will be encoded as strings. This is to avoid the - rounding that happens in Javascript otherwise. Note that this is still a - lossy operation that will not round-trip correctly and should be used - sparingly. - - If *int_as_string_bitcount* is a positive number (n), then int of size - greater than or equal to 2**n or lower than or equal to -2**n will be - encoded as strings. - - If specified, *item_sort_key* is a callable used to sort the items in - each dictionary. This is useful if you want to sort items other than - in alphabetical order by key. This option takes precedence over - *sort_keys*. - - If *sort_keys* is true (default: ``False``), the output of dictionaries - will be sorted by item. - - If *for_json* is true (default: ``False``), objects with a ``for_json()`` - method will use the return value of that method for encoding as JSON - instead of the object. - - If *ignore_nan* is true (default: ``False``), then out of range - :class:`float` values (``nan``, ``inf``, ``-inf``) will be serialized as - ``null`` in compliance with the ECMA-262 specification. If true, this will - override *allow_nan*. - - To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the - ``.default()`` method to serialize additional types), specify it with - the ``cls`` kwarg. NOTE: You should use *default* or *for_json* instead - of subclassing whenever possible. - - """ - # cached encoder - if (not skipkeys and ensure_ascii and - check_circular and not allow_nan and - cls is None and indent is None and separators is None and - encoding == 'utf-8' and default is None and use_decimal - and namedtuple_as_object and tuple_as_array and not iterable_as_array - and not bigint_as_string and not sort_keys - and not item_sort_key and not for_json - and not ignore_nan and int_as_string_bitcount is None - and not kw - ): - iterable = _default_encoder.iterencode(obj) - else: - if cls is None: - cls = JSONEncoder - iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii, - check_circular=check_circular, allow_nan=allow_nan, indent=indent, - separators=separators, encoding=encoding, - default=default, use_decimal=use_decimal, - namedtuple_as_object=namedtuple_as_object, - tuple_as_array=tuple_as_array, - iterable_as_array=iterable_as_array, - bigint_as_string=bigint_as_string, - sort_keys=sort_keys, - item_sort_key=item_sort_key, - for_json=for_json, - ignore_nan=ignore_nan, - int_as_string_bitcount=int_as_string_bitcount, - **kw).iterencode(obj) - # could accelerate with writelines in some versions of Python, at - # a debuggability cost - for chunk in iterable: - fp.write(chunk) - - -def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, - allow_nan=False, cls=None, indent=None, separators=None, - encoding='utf-8', default=None, use_decimal=True, - namedtuple_as_object=True, tuple_as_array=True, - bigint_as_string=False, sort_keys=False, item_sort_key=None, - for_json=False, ignore_nan=False, int_as_string_bitcount=None, - iterable_as_array=False, **kw): - """Serialize ``obj`` to a JSON formatted ``str``. - - If ``skipkeys`` is true then ``dict`` keys that are not basic types - (``str``, ``int``, ``long``, ``float``, ``bool``, ``None``) - will be skipped instead of raising a ``TypeError``. - - If *ensure_ascii* is false (default: ``True``), then the output may - contain non-ASCII characters, so long as they do not need to be escaped - by JSON. When it is true, all non-ASCII characters are escaped. - - If ``check_circular`` is false, then the circular reference check - for container types will be skipped and a circular reference will - result in an ``OverflowError`` (or worse). - - If *allow_nan* is true (default: ``False``), then out of range ``float`` - values (``nan``, ``inf``, ``-inf``) will be serialized to - their JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``) - instead of raising a ValueError. See - *ignore_nan* for ECMA-262 compliant behavior. - - If ``indent`` is a string, then JSON array elements and object members - will be pretty-printed with a newline followed by that string repeated - for each level of nesting. ``None`` (the default) selects the most compact - representation without any newlines. For backwards compatibility with - versions of simplejson earlier than 2.1.0, an integer is also accepted - and is converted to a string with that many spaces. - - If specified, ``separators`` should be an - ``(item_separator, key_separator)`` tuple. The default is ``(', ', ': ')`` - if *indent* is ``None`` and ``(',', ': ')`` otherwise. To get the most - compact JSON representation, you should specify ``(',', ':')`` to eliminate - whitespace. - - ``encoding`` is the character encoding for bytes instances, default is - UTF-8. - - ``default(obj)`` is a function that should return a serializable version - of obj or raise TypeError. The default simply raises TypeError. - - If *use_decimal* is true (default: ``True``) then decimal.Decimal - will be natively serialized to JSON with full precision. - - If *namedtuple_as_object* is true (default: ``True``), - :class:`tuple` subclasses with ``_asdict()`` methods will be encoded - as JSON objects. - - If *tuple_as_array* is true (default: ``True``), - :class:`tuple` (and subclasses) will be encoded as JSON arrays. - - If *iterable_as_array* is true (default: ``False``), - any object not in the above table that implements ``__iter__()`` - will be encoded as a JSON array. - - If *bigint_as_string* is true (not the default), ints 2**53 and higher - or lower than -2**53 will be encoded as strings. This is to avoid the - rounding that happens in Javascript otherwise. - - If *int_as_string_bitcount* is a positive number (n), then int of size - greater than or equal to 2**n or lower than or equal to -2**n will be - encoded as strings. - - If specified, *item_sort_key* is a callable used to sort the items in - each dictionary. This is useful if you want to sort items other than - in alphabetical order by key. This option takes precedence over - *sort_keys*. - - If *sort_keys* is true (default: ``False``), the output of dictionaries - will be sorted by item. - - If *for_json* is true (default: ``False``), objects with a ``for_json()`` - method will use the return value of that method for encoding as JSON - instead of the object. - - If *ignore_nan* is true (default: ``False``), then out of range - :class:`float` values (``nan``, ``inf``, ``-inf``) will be serialized as - ``null`` in compliance with the ECMA-262 specification. If true, this will - override *allow_nan*. - - To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the - ``.default()`` method to serialize additional types), specify it with - the ``cls`` kwarg. NOTE: You should use *default* instead of subclassing - whenever possible. - - """ - # cached encoder - if (not skipkeys and ensure_ascii and - check_circular and not allow_nan and - cls is None and indent is None and separators is None and - encoding == 'utf-8' and default is None and use_decimal - and namedtuple_as_object and tuple_as_array and not iterable_as_array - and not bigint_as_string and not sort_keys - and not item_sort_key and not for_json - and not ignore_nan and int_as_string_bitcount is None - and not kw - ): - return _default_encoder.encode(obj) - if cls is None: - cls = JSONEncoder - return cls( - skipkeys=skipkeys, ensure_ascii=ensure_ascii, - check_circular=check_circular, allow_nan=allow_nan, indent=indent, - separators=separators, encoding=encoding, default=default, - use_decimal=use_decimal, - namedtuple_as_object=namedtuple_as_object, - tuple_as_array=tuple_as_array, - iterable_as_array=iterable_as_array, - bigint_as_string=bigint_as_string, - sort_keys=sort_keys, - item_sort_key=item_sort_key, - for_json=for_json, - ignore_nan=ignore_nan, - int_as_string_bitcount=int_as_string_bitcount, - **kw).encode(obj) - - -_default_decoder = JSONDecoder() - - -def load(fp, encoding=None, cls=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, object_pairs_hook=None, - use_decimal=False, allow_nan=False, **kw): - """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing - a JSON document as `str` or `bytes`) to a Python object. - - *encoding* determines the encoding used to interpret any - `bytes` objects decoded by this instance (``'utf-8'`` by - default). It has no effect when decoding `str` objects. - - *object_hook*, if specified, will be called with the result of every - JSON object decoded and its return value will be used in place of the - given :class:`dict`. This can be used to provide custom - deserializations (e.g. to support JSON-RPC class hinting). - - *object_pairs_hook* is an optional function that will be called with - the result of any object literal decode with an ordered list of pairs. - The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders - that rely on the order that the key and value pairs are decoded (for - example, :func:`collections.OrderedDict` will remember the order of - insertion). If *object_hook* is also defined, the *object_pairs_hook* - takes priority. - - *parse_float*, if specified, will be called with the string of every - JSON float to be decoded. By default, this is equivalent to - ``float(num_str)``. This can be used to use another datatype or parser - for JSON floats (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every - JSON int to be decoded. By default, this is equivalent to - ``int(num_str)``. This can be used to use another datatype or parser - for JSON integers (e.g. :class:`float`). - - *allow_nan*, if True (default false), will allow the parser to - accept the non-standard floats ``NaN``, ``Infinity``, and ``-Infinity`` - and enable the use of the deprecated *parse_constant*. - - If *use_decimal* is true (default: ``False``) then it implies - parse_float=decimal.Decimal for parity with ``dump``. - - *parse_constant*, if specified, will be - called with one of the following strings: ``'-Infinity'``, - ``'Infinity'``, ``'NaN'``. It is not recommended to use this feature, - as it is rare to parse non-compliant JSON containing these values. - - To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` - kwarg. NOTE: You should use *object_hook* or *object_pairs_hook* instead - of subclassing whenever possible. - - """ - return loads(fp.read(), - encoding=encoding, cls=cls, object_hook=object_hook, - parse_float=parse_float, parse_int=parse_int, - parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, - use_decimal=use_decimal, allow_nan=allow_nan, **kw) - - -def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, object_pairs_hook=None, - use_decimal=False, allow_nan=False, **kw): - """Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON - document) to a Python object. - - *encoding* determines the encoding used to interpret any - :class:`bytes` objects decoded by this instance (``'utf-8'`` by - default). It has no effect when decoding :class:`unicode` objects. - - *object_hook*, if specified, will be called with the result of every - JSON object decoded and its return value will be used in place of the - given :class:`dict`. This can be used to provide custom - deserializations (e.g. to support JSON-RPC class hinting). - - *object_pairs_hook* is an optional function that will be called with - the result of any object literal decode with an ordered list of pairs. - The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders - that rely on the order that the key and value pairs are decoded (for - example, :func:`collections.OrderedDict` will remember the order of - insertion). If *object_hook* is also defined, the *object_pairs_hook* - takes priority. - - *parse_float*, if specified, will be called with the string of every - JSON float to be decoded. By default, this is equivalent to - ``float(num_str)``. This can be used to use another datatype or parser - for JSON floats (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every - JSON int to be decoded. By default, this is equivalent to - ``int(num_str)``. This can be used to use another datatype or parser - for JSON integers (e.g. :class:`float`). - - *allow_nan*, if True (default false), will allow the parser to - accept the non-standard floats ``NaN``, ``Infinity``, and ``-Infinity`` - and enable the use of the deprecated *parse_constant*. - - If *use_decimal* is true (default: ``False``) then it implies - parse_float=decimal.Decimal for parity with ``dump``. - - *parse_constant*, if specified, will be - called with one of the following strings: ``'-Infinity'``, - ``'Infinity'``, ``'NaN'``. It is not recommended to use this feature, - as it is rare to parse non-compliant JSON containing these values. - - To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` - kwarg. NOTE: You should use *object_hook* or *object_pairs_hook* instead - of subclassing whenever possible. - - """ - if (cls is None and encoding is None and object_hook is None and - parse_int is None and parse_float is None and - parse_constant is None and object_pairs_hook is None - and not use_decimal and not allow_nan and not kw): - return _default_decoder.decode(s) - if cls is None: - cls = JSONDecoder - if object_hook is not None: - kw['object_hook'] = object_hook - if object_pairs_hook is not None: - kw['object_pairs_hook'] = object_pairs_hook - if parse_float is not None: - kw['parse_float'] = parse_float - if parse_int is not None: - kw['parse_int'] = parse_int - if parse_constant is not None: - kw['parse_constant'] = parse_constant - if use_decimal: - if parse_float is not None: - raise TypeError("use_decimal=True implies parse_float=Decimal") - kw['parse_float'] = Decimal - if allow_nan: - kw['allow_nan'] = True - return cls(encoding=encoding, **kw).decode(s) - - -def _toggle_speedups(enabled): - from . import decoder as dec - from . import encoder as enc - from . import scanner as scan - c_make_encoder = _import_c_make_encoder() - if enabled: - dec.scanstring = dec.c_scanstring or dec.py_scanstring - enc.c_make_encoder = c_make_encoder - enc.encode_basestring_ascii = (enc.c_encode_basestring_ascii or - enc.py_encode_basestring_ascii) - scan.make_scanner = scan.c_make_scanner or scan.py_make_scanner - else: - dec.scanstring = dec.py_scanstring - enc.c_make_encoder = None - enc.encode_basestring_ascii = enc.py_encode_basestring_ascii - scan.make_scanner = scan.py_make_scanner - dec.make_scanner = scan.make_scanner - global _default_decoder - _default_decoder = JSONDecoder() - global _default_encoder - _default_encoder = JSONEncoder() - -def simple_first(kv): - """Helper function to pass to item_sort_key to sort simple - elements to the top, then container elements. - """ - return (isinstance(kv[1], (list, dict, tuple)), kv[0]) diff --git a/contrib/python/simplejson/py2/simplejson/_speedups.c b/contrib/python/simplejson/py2/simplejson/_speedups.c deleted file mode 100644 index 4133b9bc761..00000000000 --- a/contrib/python/simplejson/py2/simplejson/_speedups.c +++ /dev/null @@ -1,3414 +0,0 @@ -/* -*- mode: C; c-file-style: "python"; c-basic-offset: 4 -*- */ -#include "Python.h" -#include "structmember.h" - -#if PY_MAJOR_VERSION >= 3 -#define PyInt_FromSsize_t PyLong_FromSsize_t -#define PyInt_AsSsize_t PyLong_AsSsize_t -#define PyInt_Check(obj) 0 -#define PyInt_CheckExact(obj) 0 -#define JSON_UNICHR Py_UCS4 -#define JSON_InternFromString PyUnicode_InternFromString -#define PyString_GET_SIZE PyUnicode_GET_LENGTH -#define JSON_StringCheck PyUnicode_Check -#define PY2_UNUSED -#define PY3_UNUSED UNUSED -#else /* PY_MAJOR_VERSION >= 3 */ -#define PY2_UNUSED UNUSED -#define PY3_UNUSED -#define JSON_StringCheck(obj) (PyString_Check(obj) || PyUnicode_Check(obj)) -#define PyBytes_Check PyString_Check -#define PyUnicode_READY(obj) 0 -#define PyUnicode_KIND(obj) (sizeof(Py_UNICODE)) -#define PyUnicode_DATA(obj) ((void *)(PyUnicode_AS_UNICODE(obj))) -#define PyUnicode_READ(kind, data, index) ((JSON_UNICHR)((const Py_UNICODE *)(data))[(index)]) -#define PyUnicode_GET_LENGTH PyUnicode_GET_SIZE -#define JSON_UNICHR Py_UNICODE -#define JSON_InternFromString PyString_InternFromString -#endif /* PY_MAJOR_VERSION < 3 */ - -#if PY_VERSION_HEX < 0x03090000 -#if !defined(PyObject_CallNoArgs) -#define PyObject_CallNoArgs(callable) PyObject_CallFunctionObjArgs(callable, NULL); -#endif -#if !defined(PyObject_CallOneArg) -#define PyObject_CallOneArg(callable, arg) PyObject_CallFunctionObjArgs(callable, arg, NULL); -#endif -#endif /* PY_VERSION_HEX < 0x03090000 */ - -#if PY_VERSION_HEX < 0x02070000 -#if !defined(PyOS_string_to_double) -#define PyOS_string_to_double json_PyOS_string_to_double -static double -json_PyOS_string_to_double(const char *s, char **endptr, PyObject *overflow_exception); -static double -json_PyOS_string_to_double(const char *s, char **endptr, PyObject *overflow_exception) -{ - double x; - assert(endptr == NULL); - assert(overflow_exception == NULL); - PyFPE_START_PROTECT("json_PyOS_string_to_double", return -1.0;) - x = PyOS_ascii_atof(s); - PyFPE_END_PROTECT(x) - return x; -} -#endif -#endif /* PY_VERSION_HEX < 0x02070000 */ - -#if PY_VERSION_HEX < 0x02060000 -#if !defined(Py_TYPE) -#define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) -#endif -#if !defined(Py_SIZE) -#define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) -#endif -#if !defined(PyVarObject_HEAD_INIT) -#define PyVarObject_HEAD_INIT(type, size) PyObject_HEAD_INIT(type) size, -#endif -#endif /* PY_VERSION_HEX < 0x02060000 */ - -#ifdef __GNUC__ -#define UNUSED __attribute__((__unused__)) -#else -#define UNUSED -#endif - -#define DEFAULT_ENCODING "utf-8" - -#define PyScanner_Check(op) PyObject_TypeCheck(op, &PyScannerType) -#define PyScanner_CheckExact(op) (Py_TYPE(op) == &PyScannerType) -#define PyEncoder_Check(op) PyObject_TypeCheck(op, &PyEncoderType) -#define PyEncoder_CheckExact(op) (Py_TYPE(op) == &PyEncoderType) - -#define JSON_ALLOW_NAN 1 -#define JSON_IGNORE_NAN 2 - -static PyObject *JSON_Infinity = NULL; -static PyObject *JSON_NegInfinity = NULL; -static PyObject *JSON_NaN = NULL; -static PyObject *JSON_EmptyUnicode = NULL; -#if PY_MAJOR_VERSION < 3 -static PyObject *JSON_EmptyStr = NULL; -#endif - -static PyTypeObject PyScannerType; -static PyTypeObject PyEncoderType; - -typedef struct { - PyObject *large_strings; /* A list of previously accumulated large strings */ - PyObject *small_strings; /* Pending small strings */ -} JSON_Accu; - -static int -JSON_Accu_Init(JSON_Accu *acc); -static int -JSON_Accu_Accumulate(JSON_Accu *acc, PyObject *unicode); -static PyObject * -JSON_Accu_FinishAsList(JSON_Accu *acc); -static void -JSON_Accu_Destroy(JSON_Accu *acc); - -#define ERR_EXPECTING_VALUE "Expecting value" -#define ERR_ARRAY_DELIMITER "Expecting ',' delimiter or ']'" -#define ERR_ARRAY_VALUE_FIRST "Expecting value or ']'" -#define ERR_OBJECT_DELIMITER "Expecting ',' delimiter or '}'" -#define ERR_OBJECT_PROPERTY "Expecting property name enclosed in double quotes" -#define ERR_OBJECT_PROPERTY_FIRST "Expecting property name enclosed in double quotes or '}'" -#define ERR_OBJECT_PROPERTY_DELIMITER "Expecting ':' delimiter" -#define ERR_STRING_UNTERMINATED "Unterminated string starting at" -#define ERR_STRING_CONTROL "Invalid control character %r at" -#define ERR_STRING_ESC1 "Invalid \\X escape sequence %r" -#define ERR_STRING_ESC4 "Invalid \\uXXXX escape sequence" -#define FOR_JSON_METHOD_NAME "for_json" -#define ASDICT_METHOD_NAME "_asdict" - - -typedef struct _PyScannerObject { - PyObject_HEAD - PyObject *encoding; - PyObject *strict_bool; - int strict; - PyObject *object_hook; - PyObject *pairs_hook; - PyObject *parse_float; - PyObject *parse_int; - PyObject *parse_constant; - PyObject *memo; -} PyScannerObject; - -static PyMemberDef scanner_members[] = { - {"encoding", T_OBJECT, offsetof(PyScannerObject, encoding), READONLY, "encoding"}, - {"strict", T_OBJECT, offsetof(PyScannerObject, strict_bool), READONLY, "strict"}, - {"object_hook", T_OBJECT, offsetof(PyScannerObject, object_hook), READONLY, "object_hook"}, - {"object_pairs_hook", T_OBJECT, offsetof(PyScannerObject, pairs_hook), READONLY, "object_pairs_hook"}, - {"parse_float", T_OBJECT, offsetof(PyScannerObject, parse_float), READONLY, "parse_float"}, - {"parse_int", T_OBJECT, offsetof(PyScannerObject, parse_int), READONLY, "parse_int"}, - {"parse_constant", T_OBJECT, offsetof(PyScannerObject, parse_constant), READONLY, "parse_constant"}, - {NULL} -}; - -typedef struct _PyEncoderObject { - PyObject_HEAD - PyObject *markers; - PyObject *defaultfn; - PyObject *encoder; - PyObject *indent; - PyObject *key_separator; - PyObject *item_separator; - PyObject *sort_keys; - PyObject *key_memo; - PyObject *encoding; - PyObject *Decimal; - PyObject *skipkeys_bool; - int skipkeys; - int fast_encode; - /* 0, JSON_ALLOW_NAN, JSON_IGNORE_NAN */ - int allow_or_ignore_nan; - int use_decimal; - int namedtuple_as_object; - int tuple_as_array; - int iterable_as_array; - PyObject *max_long_size; - PyObject *min_long_size; - PyObject *item_sort_key; - PyObject *item_sort_kw; - int for_json; -} PyEncoderObject; - -static PyMemberDef encoder_members[] = { - {"markers", T_OBJECT, offsetof(PyEncoderObject, markers), READONLY, "markers"}, - {"default", T_OBJECT, offsetof(PyEncoderObject, defaultfn), READONLY, "default"}, - {"encoder", T_OBJECT, offsetof(PyEncoderObject, encoder), READONLY, "encoder"}, - {"encoding", T_OBJECT, offsetof(PyEncoderObject, encoder), READONLY, "encoding"}, - {"indent", T_OBJECT, offsetof(PyEncoderObject, indent), READONLY, "indent"}, - {"key_separator", T_OBJECT, offsetof(PyEncoderObject, key_separator), READONLY, "key_separator"}, - {"item_separator", T_OBJECT, offsetof(PyEncoderObject, item_separator), READONLY, "item_separator"}, - {"sort_keys", T_OBJECT, offsetof(PyEncoderObject, sort_keys), READONLY, "sort_keys"}, - /* Python 2.5 does not support T_BOOl */ - {"skipkeys", T_OBJECT, offsetof(PyEncoderObject, skipkeys_bool), READONLY, "skipkeys"}, - {"key_memo", T_OBJECT, offsetof(PyEncoderObject, key_memo), READONLY, "key_memo"}, - {"item_sort_key", T_OBJECT, offsetof(PyEncoderObject, item_sort_key), READONLY, "item_sort_key"}, - {"max_long_size", T_OBJECT, offsetof(PyEncoderObject, max_long_size), READONLY, "max_long_size"}, - {"min_long_size", T_OBJECT, offsetof(PyEncoderObject, min_long_size), READONLY, "min_long_size"}, - {NULL} -}; - -static PyObject * -join_list_unicode(PyObject *lst); -static PyObject * -JSON_ParseEncoding(PyObject *encoding); -static PyObject * -maybe_quote_bigint(PyEncoderObject* s, PyObject *encoded, PyObject *obj); -static Py_ssize_t -ascii_char_size(JSON_UNICHR c); -static Py_ssize_t -ascii_escape_char(JSON_UNICHR c, char *output, Py_ssize_t chars); -static PyObject * -ascii_escape_unicode(PyObject *pystr); -static PyObject * -ascii_escape_str(PyObject *pystr); -static PyObject * -py_encode_basestring_ascii(PyObject* self UNUSED, PyObject *pystr); -#if PY_MAJOR_VERSION < 3 -static PyObject * -join_list_string(PyObject *lst); -static PyObject * -scan_once_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr); -static PyObject * -scanstring_str(PyObject *pystr, Py_ssize_t end, char *encoding, int strict, Py_ssize_t *next_end_ptr); -static PyObject * -_parse_object_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr); -#endif -static PyObject * -scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next_end_ptr); -static PyObject * -scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr); -static PyObject * -_build_rval_index_tuple(PyObject *rval, Py_ssize_t idx); -static PyObject * -scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds); -static void -scanner_dealloc(PyObject *self); -static int -scanner_clear(PyObject *self); -static PyObject * -encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds); -static void -encoder_dealloc(PyObject *self); -static int -encoder_clear(PyObject *self); -static int -is_raw_json(PyObject *obj); -static PyObject * -encoder_stringify_key(PyEncoderObject *s, PyObject *key); -static int -encoder_listencode_list(PyEncoderObject *s, JSON_Accu *rval, PyObject *seq, Py_ssize_t indent_level); -static int -encoder_listencode_obj(PyEncoderObject *s, JSON_Accu *rval, PyObject *obj, Py_ssize_t indent_level); -static int -encoder_listencode_dict(PyEncoderObject *s, JSON_Accu *rval, PyObject *dct, Py_ssize_t indent_level); -static PyObject * -_encoded_const(PyObject *obj); -static void -raise_errmsg(char *msg, PyObject *s, Py_ssize_t end); -static PyObject * -encoder_encode_string(PyEncoderObject *s, PyObject *obj); -static int -_convertPyInt_AsSsize_t(PyObject *o, Py_ssize_t *size_ptr); -static PyObject * -_convertPyInt_FromSsize_t(Py_ssize_t *size_ptr); -static int -_call_json_method(PyObject *obj, const char *method_name, PyObject **result); -static PyObject * -encoder_encode_float(PyEncoderObject *s, PyObject *obj); -static PyObject * -moduleinit(void); - -#define S_CHAR(c) (c >= ' ' && c <= '~' && c != '\\' && c != '"') -#define IS_WHITESPACE(c) (((c) == ' ') || ((c) == '\t') || ((c) == '\n') || ((c) == '\r')) - -#define MIN_EXPANSION 6 - -static PyObject* RawJSONType = NULL; -static int -is_raw_json(PyObject *obj) -{ - return PyObject_IsInstance(obj, RawJSONType) ? 1 : 0; -} - -static int -JSON_Accu_Init(JSON_Accu *acc) -{ - /* Lazily allocated */ - acc->large_strings = NULL; - acc->small_strings = PyList_New(0); - if (acc->small_strings == NULL) - return -1; - return 0; -} - -static int -flush_accumulator(JSON_Accu *acc) -{ - Py_ssize_t nsmall = PyList_GET_SIZE(acc->small_strings); - if (nsmall) { - int ret; - PyObject *joined; - if (acc->large_strings == NULL) { - acc->large_strings = PyList_New(0); - if (acc->large_strings == NULL) - return -1; - } -#if PY_MAJOR_VERSION >= 3 - joined = join_list_unicode(acc->small_strings); -#else /* PY_MAJOR_VERSION >= 3 */ - joined = join_list_string(acc->small_strings); -#endif /* PY_MAJOR_VERSION < 3 */ - if (joined == NULL) - return -1; - if (PyList_SetSlice(acc->small_strings, 0, nsmall, NULL)) { - Py_DECREF(joined); - return -1; - } - ret = PyList_Append(acc->large_strings, joined); - Py_DECREF(joined); - return ret; - } - return 0; -} - -static int -JSON_Accu_Accumulate(JSON_Accu *acc, PyObject *unicode) -{ - Py_ssize_t nsmall; -#if PY_MAJOR_VERSION >= 3 - assert(PyUnicode_Check(unicode)); -#else /* PY_MAJOR_VERSION >= 3 */ - assert(PyString_Check(unicode) || PyUnicode_Check(unicode)); -#endif /* PY_MAJOR_VERSION < 3 */ - - if (PyList_Append(acc->small_strings, unicode)) - return -1; - nsmall = PyList_GET_SIZE(acc->small_strings); - /* Each item in a list of unicode objects has an overhead (in 64-bit - * builds) of: - * - 8 bytes for the list slot - * - 56 bytes for the header of the unicode object - * that is, 64 bytes. 100000 such objects waste more than 6MB - * compared to a single concatenated string. - */ - if (nsmall < 100000) - return 0; - return flush_accumulator(acc); -} - -static PyObject * -JSON_Accu_FinishAsList(JSON_Accu *acc) -{ - int ret; - PyObject *res; - - ret = flush_accumulator(acc); - Py_CLEAR(acc->small_strings); - if (ret) { - Py_CLEAR(acc->large_strings); - return NULL; - } - res = acc->large_strings; - acc->large_strings = NULL; - if (res == NULL) - return PyList_New(0); - return res; -} - -static void -JSON_Accu_Destroy(JSON_Accu *acc) -{ - Py_CLEAR(acc->small_strings); - Py_CLEAR(acc->large_strings); -} - -static int -IS_DIGIT(JSON_UNICHR c) -{ - return c >= '0' && c <= '9'; -} - -static PyObject * -maybe_quote_bigint(PyEncoderObject* s, PyObject *encoded, PyObject *obj) -{ - if (s->max_long_size != Py_None && s->min_long_size != Py_None) { - if (PyObject_RichCompareBool(obj, s->max_long_size, Py_GE) || - PyObject_RichCompareBool(obj, s->min_long_size, Py_LE)) { -#if PY_MAJOR_VERSION >= 3 - PyObject* quoted = PyUnicode_FromFormat("\"%U\"", encoded); -#else - PyObject* quoted = PyString_FromFormat("\"%s\"", - PyString_AsString(encoded)); -#endif - Py_DECREF(encoded); - encoded = quoted; - } - } - - return encoded; -} - -static int -_call_json_method(PyObject *obj, const char *method_name, PyObject **result) -{ - int rval = 0; - PyObject *method = PyObject_GetAttrString(obj, method_name); - if (method == NULL) { - PyErr_Clear(); - return 0; - } - if (PyCallable_Check(method)) { - PyObject *tmp = PyObject_CallNoArgs(method); - if (tmp == NULL && PyErr_ExceptionMatches(PyExc_TypeError)) { - PyErr_Clear(); - } else { - // This will set result to NULL if a TypeError occurred, - // which must be checked by the caller - *result = tmp; - rval = 1; - } - } - Py_DECREF(method); - return rval; -} - -static int -_convertPyInt_AsSsize_t(PyObject *o, Py_ssize_t *size_ptr) -{ - /* PyObject to Py_ssize_t converter */ - *size_ptr = PyInt_AsSsize_t(o); - if (*size_ptr == -1 && PyErr_Occurred()) - return 0; - return 1; -} - -static PyObject * -_convertPyInt_FromSsize_t(Py_ssize_t *size_ptr) -{ - /* Py_ssize_t to PyObject converter */ - return PyInt_FromSsize_t(*size_ptr); -} - -static Py_ssize_t -ascii_escape_char(JSON_UNICHR c, char *output, Py_ssize_t chars) -{ - /* Escape unicode code point c to ASCII escape sequences - in char *output. output must have at least 12 bytes unused to - accommodate an escaped surrogate pair "\uXXXX\uXXXX" */ - if (S_CHAR(c)) { - output[chars++] = (char)c; - } - else { - output[chars++] = '\\'; - switch (c) { - case '\\': output[chars++] = (char)c; break; - case '"': output[chars++] = (char)c; break; - case '\b': output[chars++] = 'b'; break; - case '\f': output[chars++] = 'f'; break; - case '\n': output[chars++] = 'n'; break; - case '\r': output[chars++] = 'r'; break; - case '\t': output[chars++] = 't'; break; - default: -#if PY_MAJOR_VERSION >= 3 || defined(Py_UNICODE_WIDE) - if (c >= 0x10000) { - /* UTF-16 surrogate pair */ - JSON_UNICHR v = c - 0x10000; - c = 0xd800 | ((v >> 10) & 0x3ff); - output[chars++] = 'u'; - output[chars++] = "0123456789abcdef"[(c >> 12) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 8) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 4) & 0xf]; - output[chars++] = "0123456789abcdef"[(c ) & 0xf]; - c = 0xdc00 | (v & 0x3ff); - output[chars++] = '\\'; - } -#endif - output[chars++] = 'u'; - output[chars++] = "0123456789abcdef"[(c >> 12) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 8) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 4) & 0xf]; - output[chars++] = "0123456789abcdef"[(c ) & 0xf]; - } - } - return chars; -} - -static Py_ssize_t -ascii_char_size(JSON_UNICHR c) -{ - if (S_CHAR(c)) { - return 1; - } - else if (c == '\\' || - c == '"' || - c == '\b' || - c == '\f' || - c == '\n' || - c == '\r' || - c == '\t') { - return 2; - } -#if PY_MAJOR_VERSION >= 3 || defined(Py_UNICODE_WIDE) - else if (c >= 0x10000U) { - return 2 * MIN_EXPANSION; - } -#endif - else { - return MIN_EXPANSION; - } -} - -static PyObject * -ascii_escape_unicode(PyObject *pystr) -{ - /* Take a PyUnicode pystr and return a new ASCII-only escaped PyString */ - Py_ssize_t i; - Py_ssize_t input_chars = PyUnicode_GET_LENGTH(pystr); - Py_ssize_t output_size = 2; - Py_ssize_t chars; - PY2_UNUSED int kind = PyUnicode_KIND(pystr); - void *data = PyUnicode_DATA(pystr); - PyObject *rval; - char *output; - - output_size = 2; - for (i = 0; i < input_chars; i++) { - output_size += ascii_char_size(PyUnicode_READ(kind, data, i)); - } -#if PY_MAJOR_VERSION >= 3 - rval = PyUnicode_New(output_size, 127); - if (rval == NULL) { - return NULL; - } - assert(PyUnicode_KIND(rval) == PyUnicode_1BYTE_KIND); - output = (char *)PyUnicode_DATA(rval); -#else - rval = PyString_FromStringAndSize(NULL, output_size); - if (rval == NULL) { - return NULL; - } - output = PyString_AS_STRING(rval); -#endif - chars = 0; - output[chars++] = '"'; - for (i = 0; i < input_chars; i++) { - chars = ascii_escape_char(PyUnicode_READ(kind, data, i), output, chars); - } - output[chars++] = '"'; - assert(chars == output_size); - return rval; -} - -#if PY_MAJOR_VERSION >= 3 - -static PyObject * -ascii_escape_str(PyObject *pystr) -{ - PyObject *rval; - PyObject *input = PyUnicode_DecodeUTF8(PyBytes_AS_STRING(pystr), PyBytes_GET_SIZE(pystr), NULL); - if (input == NULL) - return NULL; - rval = ascii_escape_unicode(input); - Py_DECREF(input); - return rval; -} - -#else /* PY_MAJOR_VERSION >= 3 */ - -static PyObject * -ascii_escape_str(PyObject *pystr) -{ - /* Take a PyString pystr and return a new ASCII-only escaped PyString */ - Py_ssize_t i; - Py_ssize_t input_chars; - Py_ssize_t output_size; - Py_ssize_t chars; - PyObject *rval; - char *output; - char *input_str; - - input_chars = PyString_GET_SIZE(pystr); - input_str = PyString_AS_STRING(pystr); - output_size = 2; - - /* Fast path for a string that's already ASCII */ - for (i = 0; i < input_chars; i++) { - JSON_UNICHR c = (JSON_UNICHR)input_str[i]; - if (c > 0x7f) { - /* We hit a non-ASCII character, bail to unicode mode */ - PyObject *uni; - uni = PyUnicode_DecodeUTF8(input_str, input_chars, "strict"); - if (uni == NULL) { - return NULL; - } - rval = ascii_escape_unicode(uni); - Py_DECREF(uni); - return rval; - } - output_size += ascii_char_size(c); - } - - rval = PyString_FromStringAndSize(NULL, output_size); - if (rval == NULL) { - return NULL; - } - chars = 0; - output = PyString_AS_STRING(rval); - output[chars++] = '"'; - for (i = 0; i < input_chars; i++) { - chars = ascii_escape_char((JSON_UNICHR)input_str[i], output, chars); - } - output[chars++] = '"'; - assert(chars == output_size); - return rval; -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -encoder_stringify_key(PyEncoderObject *s, PyObject *key) -{ - if (PyUnicode_Check(key)) { - Py_INCREF(key); - return key; - } -#if PY_MAJOR_VERSION >= 3 - else if (PyBytes_Check(key) && s->encoding != NULL) { - const char *encoding = PyUnicode_AsUTF8(s->encoding); - if (encoding == NULL) - return NULL; - return PyUnicode_Decode( - PyBytes_AS_STRING(key), - PyBytes_GET_SIZE(key), - encoding, - NULL); - } -#else /* PY_MAJOR_VERSION >= 3 */ - else if (PyString_Check(key)) { - Py_INCREF(key); - return key; - } -#endif /* PY_MAJOR_VERSION < 3 */ - else if (PyFloat_Check(key)) { - return encoder_encode_float(s, key); - } - else if (key == Py_True || key == Py_False || key == Py_None) { - /* This must come before the PyInt_Check because - True and False are also 1 and 0.*/ - return _encoded_const(key); - } - else if (PyInt_Check(key) || PyLong_Check(key)) { - if (!(PyInt_CheckExact(key) || PyLong_CheckExact(key))) { - /* See #118, do not trust custom str/repr */ - PyObject *res; - PyObject *tmp = PyObject_CallOneArg((PyObject *)&PyLong_Type, key); - if (tmp == NULL) { - return NULL; - } - res = PyObject_Str(tmp); - Py_DECREF(tmp); - return res; - } - else { - return PyObject_Str(key); - } - } - else if (s->use_decimal && PyObject_TypeCheck(key, (PyTypeObject *)s->Decimal)) { - return PyObject_Str(key); - } - if (s->skipkeys) { - Py_INCREF(Py_None); - return Py_None; - } - PyErr_Format(PyExc_TypeError, - "keys must be str, int, float, bool or None, " - "not %.100s", key->ob_type->tp_name); - return NULL; -} - -static PyObject * -encoder_dict_iteritems(PyEncoderObject *s, PyObject *dct) -{ - PyObject *items; - PyObject *iter = NULL; - PyObject *lst = NULL; - PyObject *item = NULL; - PyObject *kstr = NULL; - PyObject *sortfun = NULL; - PyObject *sortres; - static PyObject *sortargs = NULL; - - if (sortargs == NULL) { - sortargs = PyTuple_New(0); - if (sortargs == NULL) - return NULL; - } - - if (PyDict_CheckExact(dct)) - items = PyDict_Items(dct); - else - items = PyMapping_Items(dct); - if (items == NULL) - return NULL; - iter = PyObject_GetIter(items); - Py_DECREF(items); - if (iter == NULL) - return NULL; - if (s->item_sort_kw == Py_None) - return iter; - lst = PyList_New(0); - if (lst == NULL) - goto bail; - while ((item = PyIter_Next(iter))) { - PyObject *key, *value; - if (!PyTuple_Check(item) || Py_SIZE(item) != 2) { - PyErr_SetString(PyExc_ValueError, "items must return 2-tuples"); - goto bail; - } - key = PyTuple_GET_ITEM(item, 0); - if (key == NULL) - goto bail; -#if PY_MAJOR_VERSION < 3 - else if (PyString_Check(key)) { - /* item can be added as-is */ - } -#endif /* PY_MAJOR_VERSION < 3 */ - else if (PyUnicode_Check(key)) { - /* item can be added as-is */ - } - else { - PyObject *tpl; - kstr = encoder_stringify_key(s, key); - if (kstr == NULL) - goto bail; - else if (kstr == Py_None) { - /* skipkeys */ - Py_DECREF(kstr); - continue; - } - value = PyTuple_GET_ITEM(item, 1); - if (value == NULL) - goto bail; - tpl = PyTuple_Pack(2, kstr, value); - if (tpl == NULL) - goto bail; - Py_CLEAR(kstr); - Py_DECREF(item); - item = tpl; - } - if (PyList_Append(lst, item)) - goto bail; - Py_DECREF(item); - } - Py_CLEAR(iter); - if (PyErr_Occurred()) - goto bail; - sortfun = PyObject_GetAttrString(lst, "sort"); - if (sortfun == NULL) - goto bail; - sortres = PyObject_Call(sortfun, sortargs, s->item_sort_kw); - if (!sortres) - goto bail; - Py_DECREF(sortres); - Py_CLEAR(sortfun); - iter = PyObject_GetIter(lst); - Py_CLEAR(lst); - return iter; -bail: - Py_XDECREF(sortfun); - Py_XDECREF(kstr); - Py_XDECREF(item); - Py_XDECREF(lst); - Py_XDECREF(iter); - return NULL; -} - -/* Use JSONDecodeError exception to raise a nice looking ValueError subclass */ -static PyObject *JSONDecodeError = NULL; -static void -raise_errmsg(char *msg, PyObject *s, Py_ssize_t end) -{ - PyObject *exc = PyObject_CallFunction(JSONDecodeError, "(zOO&)", msg, s, _convertPyInt_FromSsize_t, &end); - if (exc) { - PyErr_SetObject(JSONDecodeError, exc); - Py_DECREF(exc); - } -} - -static PyObject * -join_list_unicode(PyObject *lst) -{ - /* return u''.join(lst) */ - return PyUnicode_Join(JSON_EmptyUnicode, lst); -} - -#if PY_MAJOR_VERSION >= 3 -#define join_list_string join_list_unicode -#else /* PY_MAJOR_VERSION >= 3 */ -static PyObject * -join_list_string(PyObject *lst) -{ - /* return ''.join(lst) */ - static PyObject *joinfn = NULL; - if (joinfn == NULL) { - joinfn = PyObject_GetAttrString(JSON_EmptyStr, "join"); - if (joinfn == NULL) - return NULL; - } - return PyObject_CallOneArg(joinfn, lst); -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -_build_rval_index_tuple(PyObject *rval, Py_ssize_t idx) -{ - /* return (rval, idx) tuple, stealing reference to rval */ - PyObject *tpl; - PyObject *pyidx; - /* - steal a reference to rval, returns (rval, idx) - */ - if (rval == NULL) { - assert(PyErr_Occurred()); - return NULL; - } - pyidx = PyInt_FromSsize_t(idx); - if (pyidx == NULL) { - Py_DECREF(rval); - return NULL; - } - tpl = PyTuple_New(2); - if (tpl == NULL) { - Py_DECREF(pyidx); - Py_DECREF(rval); - return NULL; - } - PyTuple_SET_ITEM(tpl, 0, rval); - PyTuple_SET_ITEM(tpl, 1, pyidx); - return tpl; -} - -#define APPEND_OLD_CHUNK \ - if (chunk != NULL) { \ - if (chunks == NULL) { \ - chunks = PyList_New(0); \ - if (chunks == NULL) { \ - goto bail; \ - } \ - } \ - if (PyList_Append(chunks, chunk)) { \ - goto bail; \ - } \ - Py_CLEAR(chunk); \ - } - -#if PY_MAJOR_VERSION < 3 -static PyObject * -scanstring_str(PyObject *pystr, Py_ssize_t end, char *encoding, int strict, Py_ssize_t *next_end_ptr) -{ - /* Read the JSON string from PyString pystr. - end is the index of the first character after the quote. - encoding is the encoding of pystr (must be an ASCII superset) - if strict is zero then literal control characters are allowed - *next_end_ptr is a return-by-reference index of the character - after the end quote - - Return value is a new PyString (if ASCII-only) or PyUnicode - */ - PyObject *rval; - Py_ssize_t len = PyString_GET_SIZE(pystr); - Py_ssize_t begin = end - 1; - Py_ssize_t next = begin; - int has_unicode = 0; - char *buf = PyString_AS_STRING(pystr); - PyObject *chunks = NULL; - PyObject *chunk = NULL; - PyObject *strchunk = NULL; - - if (len == end) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - else if (end < 0 || len < end) { - PyErr_SetString(PyExc_ValueError, "end is out of bounds"); - goto bail; - } - while (1) { - /* Find the end of the string or the next escape */ - Py_UNICODE c = 0; - for (next = end; next < len; next++) { - c = (unsigned char)buf[next]; - if (c == '"' || c == '\\') { - break; - } - else if (strict && c <= 0x1f) { - raise_errmsg(ERR_STRING_CONTROL, pystr, next); - goto bail; - } - else if (c > 0x7f) { - has_unicode = 1; - } - } - if (!(c == '"' || c == '\\')) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - /* Pick up this chunk if it's not zero length */ - if (next != end) { - APPEND_OLD_CHUNK - strchunk = PyString_FromStringAndSize(&buf[end], next - end); - if (strchunk == NULL) { - goto bail; - } - if (has_unicode) { - chunk = PyUnicode_FromEncodedObject(strchunk, encoding, NULL); - Py_DECREF(strchunk); - if (chunk == NULL) { - goto bail; - } - } - else { - chunk = strchunk; - } - } - next++; - if (c == '"') { - end = next; - break; - } - if (next == len) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - c = buf[next]; - if (c != 'u') { - /* Non-unicode backslash escapes */ - end = next + 1; - switch (c) { - case '"': break; - case '\\': break; - case '/': break; - case 'b': c = '\b'; break; - case 'f': c = '\f'; break; - case 'n': c = '\n'; break; - case 'r': c = '\r'; break; - case 't': c = '\t'; break; - default: c = 0; - } - if (c == 0) { - raise_errmsg(ERR_STRING_ESC1, pystr, end - 2); - goto bail; - } - } - else { - c = 0; - next++; - end = next + 4; - if (end >= len) { - raise_errmsg(ERR_STRING_ESC4, pystr, next - 1); - goto bail; - } - /* Decode 4 hex digits */ - for (; next < end; next++) { - JSON_UNICHR digit = (JSON_UNICHR)buf[next]; - c <<= 4; - switch (digit) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - c |= (digit - '0'); break; - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': - c |= (digit - 'a' + 10); break; - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': - c |= (digit - 'A' + 10); break; - default: - raise_errmsg(ERR_STRING_ESC4, pystr, end - 5); - goto bail; - } - } -#if defined(Py_UNICODE_WIDE) - /* Surrogate pair */ - if ((c & 0xfc00) == 0xd800) { - if (end + 6 < len && buf[next] == '\\' && buf[next+1] == 'u') { - JSON_UNICHR c2 = 0; - end += 6; - /* Decode 4 hex digits */ - for (next += 2; next < end; next++) { - c2 <<= 4; - JSON_UNICHR digit = buf[next]; - switch (digit) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - c2 |= (digit - '0'); break; - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': - c2 |= (digit - 'a' + 10); break; - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': - c2 |= (digit - 'A' + 10); break; - default: - raise_errmsg(ERR_STRING_ESC4, pystr, end - 5); - goto bail; - } - } - if ((c2 & 0xfc00) != 0xdc00) { - /* not a low surrogate, rewind */ - end -= 6; - next = end; - } - else { - c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00)); - } - } - } -#endif /* Py_UNICODE_WIDE */ - } - if (c > 0x7f) { - has_unicode = 1; - } - APPEND_OLD_CHUNK - if (has_unicode) { - chunk = PyUnicode_FromOrdinal(c); - if (chunk == NULL) { - goto bail; - } - } - else { - char c_char = Py_CHARMASK(c); - chunk = PyString_FromStringAndSize(&c_char, 1); - if (chunk == NULL) { - goto bail; - } - } - } - - if (chunks == NULL) { - if (chunk != NULL) - rval = chunk; - else { - rval = JSON_EmptyStr; - Py_INCREF(rval); - } - } - else { - APPEND_OLD_CHUNK - rval = join_list_string(chunks); - if (rval == NULL) { - goto bail; - } - Py_CLEAR(chunks); - } - - *next_end_ptr = end; - return rval; -bail: - *next_end_ptr = -1; - Py_XDECREF(chunk); - Py_XDECREF(chunks); - return NULL; -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next_end_ptr) -{ - /* Read the JSON string from PyUnicode pystr. - end is the index of the first character after the quote. - if strict is zero then literal control characters are allowed - *next_end_ptr is a return-by-reference index of the character - after the end quote - - Return value is a new PyUnicode - */ - PyObject *rval; - Py_ssize_t begin = end - 1; - Py_ssize_t next = begin; - PY2_UNUSED int kind = PyUnicode_KIND(pystr); - Py_ssize_t len = PyUnicode_GET_LENGTH(pystr); - void *buf = PyUnicode_DATA(pystr); - PyObject *chunks = NULL; - PyObject *chunk = NULL; - - if (len == end) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - else if (end < 0 || len < end) { - PyErr_SetString(PyExc_ValueError, "end is out of bounds"); - goto bail; - } - while (1) { - /* Find the end of the string or the next escape */ - JSON_UNICHR c = 0; - for (next = end; next < len; next++) { - c = PyUnicode_READ(kind, buf, next); - if (c == '"' || c == '\\') { - break; - } - else if (strict && c <= 0x1f) { - raise_errmsg(ERR_STRING_CONTROL, pystr, next); - goto bail; - } - } - if (!(c == '"' || c == '\\')) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - /* Pick up this chunk if it's not zero length */ - if (next != end) { - APPEND_OLD_CHUNK -#if PY_MAJOR_VERSION < 3 - chunk = PyUnicode_FromUnicode(&((const Py_UNICODE *)buf)[end], next - end); -#else - chunk = PyUnicode_Substring(pystr, end, next); -#endif - if (chunk == NULL) { - goto bail; - } - } - next++; - if (c == '"') { - end = next; - break; - } - if (next == len) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - c = PyUnicode_READ(kind, buf, next); - if (c != 'u') { - /* Non-unicode backslash escapes */ - end = next + 1; - switch (c) { - case '"': break; - case '\\': break; - case '/': break; - case 'b': c = '\b'; break; - case 'f': c = '\f'; break; - case 'n': c = '\n'; break; - case 'r': c = '\r'; break; - case 't': c = '\t'; break; - default: c = 0; - } - if (c == 0) { - raise_errmsg(ERR_STRING_ESC1, pystr, end - 2); - goto bail; - } - } - else { - c = 0; - next++; - end = next + 4; - if (end >= len) { - raise_errmsg(ERR_STRING_ESC4, pystr, next - 1); - goto bail; - } - /* Decode 4 hex digits */ - for (; next < end; next++) { - JSON_UNICHR digit = PyUnicode_READ(kind, buf, next); - c <<= 4; - switch (digit) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - c |= (digit - '0'); break; - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': - c |= (digit - 'a' + 10); break; - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': - c |= (digit - 'A' + 10); break; - default: - raise_errmsg(ERR_STRING_ESC4, pystr, end - 5); - goto bail; - } - } -#if PY_MAJOR_VERSION >= 3 || defined(Py_UNICODE_WIDE) - /* Surrogate pair */ - if ((c & 0xfc00) == 0xd800) { - JSON_UNICHR c2 = 0; - if (end + 6 < len && - PyUnicode_READ(kind, buf, next) == '\\' && - PyUnicode_READ(kind, buf, next + 1) == 'u') { - end += 6; - /* Decode 4 hex digits */ - for (next += 2; next < end; next++) { - JSON_UNICHR digit = PyUnicode_READ(kind, buf, next); - c2 <<= 4; - switch (digit) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - c2 |= (digit - '0'); break; - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': - c2 |= (digit - 'a' + 10); break; - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': - c2 |= (digit - 'A' + 10); break; - default: - raise_errmsg(ERR_STRING_ESC4, pystr, end - 5); - goto bail; - } - } - if ((c2 & 0xfc00) != 0xdc00) { - /* not a low surrogate, rewind */ - end -= 6; - next = end; - } - else { - c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00)); - } - } - } -#endif - } - APPEND_OLD_CHUNK - chunk = PyUnicode_FromOrdinal(c); - if (chunk == NULL) { - goto bail; - } - } - - if (chunks == NULL) { - if (chunk != NULL) - rval = chunk; - else { - rval = JSON_EmptyUnicode; - Py_INCREF(rval); - } - } - else { - APPEND_OLD_CHUNK - rval = join_list_unicode(chunks); - if (rval == NULL) { - goto bail; - } - Py_CLEAR(chunks); - } - *next_end_ptr = end; - return rval; -bail: - *next_end_ptr = -1; - Py_XDECREF(chunk); - Py_XDECREF(chunks); - return NULL; -} - -PyDoc_STRVAR(pydoc_scanstring, - "scanstring(basestring, end, encoding, strict=True) -> (str, end)\n" - "\n" - "Scan the string s for a JSON string. End is the index of the\n" - "character in s after the quote that started the JSON string.\n" - "Unescapes all valid JSON string escape sequences and raises ValueError\n" - "on attempt to decode an invalid string. If strict is False then literal\n" - "control characters are allowed in the string.\n" - "\n" - "Returns a tuple of the decoded string and the index of the character in s\n" - "after the end quote." -); - -static PyObject * -py_scanstring(PyObject* self UNUSED, PyObject *args) -{ - PyObject *pystr; - PyObject *rval; - Py_ssize_t end; - Py_ssize_t next_end = -1; - char *encoding = NULL; - int strict = 1; - if (!PyArg_ParseTuple(args, "OO&|zi:scanstring", &pystr, _convertPyInt_AsSsize_t, &end, &encoding, &strict)) { - return NULL; - } - if (encoding == NULL) { - encoding = DEFAULT_ENCODING; - } - if (PyUnicode_Check(pystr)) { - if (PyUnicode_READY(pystr)) - return NULL; - rval = scanstring_unicode(pystr, end, strict, &next_end); - } -#if PY_MAJOR_VERSION < 3 - /* Using a bytes input is unsupported for scanning in Python 3. - It is coerced to str in the decoder before it gets here. */ - else if (PyString_Check(pystr)) { - rval = scanstring_str(pystr, end, encoding, strict, &next_end); - } -#endif - else { - PyErr_Format(PyExc_TypeError, - "first argument must be a string, not %.80s", - Py_TYPE(pystr)->tp_name); - return NULL; - } - return _build_rval_index_tuple(rval, next_end); -} - -PyDoc_STRVAR(pydoc_encode_basestring_ascii, - "encode_basestring_ascii(basestring) -> str\n" - "\n" - "Return an ASCII-only JSON representation of a Python string" -); - -static PyObject * -py_encode_basestring_ascii(PyObject* self UNUSED, PyObject *pystr) -{ - /* Return an ASCII-only JSON representation of a Python string */ - /* METH_O */ - if (PyBytes_Check(pystr)) { - return ascii_escape_str(pystr); - } - else if (PyUnicode_Check(pystr)) { - if (PyUnicode_READY(pystr)) - return NULL; - return ascii_escape_unicode(pystr); - } - else { - PyErr_Format(PyExc_TypeError, - "first argument must be a string, not %.80s", - Py_TYPE(pystr)->tp_name); - return NULL; - } -} - -static void -scanner_dealloc(PyObject *self) -{ - /* bpo-31095: UnTrack is needed before calling any callbacks */ - PyObject_GC_UnTrack(self); - scanner_clear(self); - Py_TYPE(self)->tp_free(self); -} - -static int -scanner_traverse(PyObject *self, visitproc visit, void *arg) -{ - PyScannerObject *s; - assert(PyScanner_Check(self)); - s = (PyScannerObject *)self; - Py_VISIT(s->encoding); - Py_VISIT(s->strict_bool); - Py_VISIT(s->object_hook); - Py_VISIT(s->pairs_hook); - Py_VISIT(s->parse_float); - Py_VISIT(s->parse_int); - Py_VISIT(s->parse_constant); - Py_VISIT(s->memo); - return 0; -} - -static int -scanner_clear(PyObject *self) -{ - PyScannerObject *s; - assert(PyScanner_Check(self)); - s = (PyScannerObject *)self; - Py_CLEAR(s->encoding); - Py_CLEAR(s->strict_bool); - Py_CLEAR(s->object_hook); - Py_CLEAR(s->pairs_hook); - Py_CLEAR(s->parse_float); - Py_CLEAR(s->parse_int); - Py_CLEAR(s->parse_constant); - Py_CLEAR(s->memo); - return 0; -} - -#if PY_MAJOR_VERSION < 3 -static PyObject * -_parse_object_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON object from PyString pystr. - idx is the index of the first character after the opening curly brace. - *next_idx_ptr is a return-by-reference index to the first character after - the closing curly brace. - - Returns a new PyObject (usually a dict, but object_hook or - object_pairs_hook can change that) - */ - char *str = PyString_AS_STRING(pystr); - Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1; - PyObject *rval = NULL; - PyObject *pairs = NULL; - PyObject *item; - PyObject *key = NULL; - PyObject *val = NULL; - char *encoding = PyString_AS_STRING(s->encoding); - int has_pairs_hook = (s->pairs_hook != Py_None); - int did_parse = 0; - Py_ssize_t next_idx; - if (has_pairs_hook) { - pairs = PyList_New(0); - if (pairs == NULL) - return NULL; - } - else { - rval = PyDict_New(); - if (rval == NULL) - return NULL; - } - - /* skip whitespace after { */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* only loop if the object is non-empty */ - if (idx <= end_idx && str[idx] != '}') { - int trailing_delimiter = 0; - while (idx <= end_idx) { - PyObject *memokey; - trailing_delimiter = 0; - - /* read key */ - if (str[idx] != '"') { - raise_errmsg(ERR_OBJECT_PROPERTY, pystr, idx); - goto bail; - } - key = scanstring_str(pystr, idx + 1, encoding, s->strict, &next_idx); - if (key == NULL) - goto bail; - memokey = PyDict_GetItem(s->memo, key); - if (memokey != NULL) { - Py_INCREF(memokey); - Py_DECREF(key); - key = memokey; - } - else { - if (PyDict_SetItem(s->memo, key, key) < 0) - goto bail; - } - idx = next_idx; - - /* skip whitespace between key and : delimiter, read :, skip whitespace */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - if (idx > end_idx || str[idx] != ':') { - raise_errmsg(ERR_OBJECT_PROPERTY_DELIMITER, pystr, idx); - goto bail; - } - idx++; - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* read any JSON data type */ - val = scan_once_str(s, pystr, idx, &next_idx); - if (val == NULL) - goto bail; - - if (has_pairs_hook) { - item = PyTuple_Pack(2, key, val); - if (item == NULL) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - if (PyList_Append(pairs, item) == -1) { - Py_DECREF(item); - goto bail; - } - Py_DECREF(item); - } - else { - if (PyDict_SetItem(rval, key, val) < 0) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - } - idx = next_idx; - - /* skip whitespace before } or , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* bail if the object is closed or we didn't get the , delimiter */ - did_parse = 1; - if (idx > end_idx) break; - if (str[idx] == '}') { - break; - } - else if (str[idx] != ',') { - raise_errmsg(ERR_OBJECT_DELIMITER, pystr, idx); - goto bail; - } - idx++; - - /* skip whitespace after , delimiter */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - trailing_delimiter = 1; - } - if (trailing_delimiter) { - raise_errmsg(ERR_OBJECT_PROPERTY, pystr, idx); - goto bail; - } - } - /* verify that idx < end_idx, str[idx] should be '}' */ - if (idx > end_idx || str[idx] != '}') { - if (did_parse) { - raise_errmsg(ERR_OBJECT_DELIMITER, pystr, idx); - } else { - raise_errmsg(ERR_OBJECT_PROPERTY_FIRST, pystr, idx); - } - goto bail; - } - - /* if pairs_hook is not None: rval = object_pairs_hook(pairs) */ - if (s->pairs_hook != Py_None) { - val = PyObject_CallOneArg(s->pairs_hook, pairs); - if (val == NULL) - goto bail; - Py_DECREF(pairs); - *next_idx_ptr = idx + 1; - return val; - } - - /* if object_hook is not None: rval = object_hook(rval) */ - if (s->object_hook != Py_None) { - val = PyObject_CallOneArg(s->object_hook, rval); - if (val == NULL) - goto bail; - Py_DECREF(rval); - rval = val; - val = NULL; - } - *next_idx_ptr = idx + 1; - return rval; -bail: - Py_XDECREF(rval); - Py_XDECREF(key); - Py_XDECREF(val); - Py_XDECREF(pairs); - return NULL; -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -_parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON object from PyUnicode pystr. - idx is the index of the first character after the opening curly brace. - *next_idx_ptr is a return-by-reference index to the first character after - the closing curly brace. - - Returns a new PyObject (usually a dict, but object_hook can change that) - */ - void *str = PyUnicode_DATA(pystr); - Py_ssize_t end_idx = PyUnicode_GET_LENGTH(pystr) - 1; - PY2_UNUSED int kind = PyUnicode_KIND(pystr); - PyObject *rval = NULL; - PyObject *pairs = NULL; - PyObject *item; - PyObject *key = NULL; - PyObject *val = NULL; - int has_pairs_hook = (s->pairs_hook != Py_None); - int did_parse = 0; - Py_ssize_t next_idx; - - if (has_pairs_hook) { - pairs = PyList_New(0); - if (pairs == NULL) - return NULL; - } - else { - rval = PyDict_New(); - if (rval == NULL) - return NULL; - } - - /* skip whitespace after { */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - - /* only loop if the object is non-empty */ - if (idx <= end_idx && PyUnicode_READ(kind, str, idx) != '}') { - int trailing_delimiter = 0; - while (idx <= end_idx) { - PyObject *memokey; - trailing_delimiter = 0; - - /* read key */ - if (PyUnicode_READ(kind, str, idx) != '"') { - raise_errmsg(ERR_OBJECT_PROPERTY, pystr, idx); - goto bail; - } - key = scanstring_unicode(pystr, idx + 1, s->strict, &next_idx); - if (key == NULL) - goto bail; - memokey = PyDict_GetItem(s->memo, key); - if (memokey != NULL) { - Py_INCREF(memokey); - Py_DECREF(key); - key = memokey; - } - else { - if (PyDict_SetItem(s->memo, key, key) < 0) - goto bail; - } - idx = next_idx; - - /* skip whitespace between key and : delimiter, read :, skip - whitespace */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - if (idx > end_idx || PyUnicode_READ(kind, str, idx) != ':') { - raise_errmsg(ERR_OBJECT_PROPERTY_DELIMITER, pystr, idx); - goto bail; - } - idx++; - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - - /* read any JSON term */ - val = scan_once_unicode(s, pystr, idx, &next_idx); - if (val == NULL) - goto bail; - - if (has_pairs_hook) { - item = PyTuple_Pack(2, key, val); - if (item == NULL) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - if (PyList_Append(pairs, item) == -1) { - Py_DECREF(item); - goto bail; - } - Py_DECREF(item); - } - else { - if (PyDict_SetItem(rval, key, val) < 0) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - } - idx = next_idx; - - /* skip whitespace before } or , */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - - /* bail if the object is closed or we didn't get the , - delimiter */ - did_parse = 1; - if (idx > end_idx) break; - if (PyUnicode_READ(kind, str, idx) == '}') { - break; - } - else if (PyUnicode_READ(kind, str, idx) != ',') { - raise_errmsg(ERR_OBJECT_DELIMITER, pystr, idx); - goto bail; - } - idx++; - - /* skip whitespace after , delimiter */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - trailing_delimiter = 1; - } - if (trailing_delimiter) { - raise_errmsg(ERR_OBJECT_PROPERTY, pystr, idx); - goto bail; - } - } - - /* verify that idx < end_idx, str[idx] should be '}' */ - if (idx > end_idx || PyUnicode_READ(kind, str, idx) != '}') { - if (did_parse) { - raise_errmsg(ERR_OBJECT_DELIMITER, pystr, idx); - } else { - raise_errmsg(ERR_OBJECT_PROPERTY_FIRST, pystr, idx); - } - goto bail; - } - - /* if pairs_hook is not None: rval = object_pairs_hook(pairs) */ - if (s->pairs_hook != Py_None) { - val = PyObject_CallOneArg(s->pairs_hook, pairs); - if (val == NULL) - goto bail; - Py_DECREF(pairs); - *next_idx_ptr = idx + 1; - return val; - } - - /* if object_hook is not None: rval = object_hook(rval) */ - if (s->object_hook != Py_None) { - val = PyObject_CallOneArg(s->object_hook, rval); - if (val == NULL) - goto bail; - Py_DECREF(rval); - rval = val; - val = NULL; - } - *next_idx_ptr = idx + 1; - return rval; -bail: - Py_XDECREF(rval); - Py_XDECREF(key); - Py_XDECREF(val); - Py_XDECREF(pairs); - return NULL; -} - -#if PY_MAJOR_VERSION < 3 -static PyObject * -_parse_array_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON array from PyString pystr. - idx is the index of the first character after the opening brace. - *next_idx_ptr is a return-by-reference index to the first character after - the closing brace. - - Returns a new PyList - */ - char *str = PyString_AS_STRING(pystr); - Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1; - PyObject *val = NULL; - PyObject *rval = PyList_New(0); - Py_ssize_t next_idx; - if (rval == NULL) - return NULL; - - /* skip whitespace after [ */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* only loop if the array is non-empty */ - if (idx <= end_idx && str[idx] != ']') { - int trailing_delimiter = 0; - while (idx <= end_idx) { - trailing_delimiter = 0; - /* read any JSON term and de-tuplefy the (rval, idx) */ - val = scan_once_str(s, pystr, idx, &next_idx); - if (val == NULL) { - goto bail; - } - - if (PyList_Append(rval, val) == -1) - goto bail; - - Py_CLEAR(val); - idx = next_idx; - - /* skip whitespace between term and , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* bail if the array is closed or we didn't get the , delimiter */ - if (idx > end_idx) break; - if (str[idx] == ']') { - break; - } - else if (str[idx] != ',') { - raise_errmsg(ERR_ARRAY_DELIMITER, pystr, idx); - goto bail; - } - idx++; - - /* skip whitespace after , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - trailing_delimiter = 1; - } - if (trailing_delimiter) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx); - goto bail; - } - } - - /* verify that idx < end_idx, str[idx] should be ']' */ - if (idx > end_idx || str[idx] != ']') { - if (PyList_GET_SIZE(rval)) { - raise_errmsg(ERR_ARRAY_DELIMITER, pystr, idx); - } else { - raise_errmsg(ERR_ARRAY_VALUE_FIRST, pystr, idx); - } - goto bail; - } - *next_idx_ptr = idx + 1; - return rval; -bail: - Py_XDECREF(val); - Py_DECREF(rval); - return NULL; -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -_parse_array_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON array from PyString pystr. - idx is the index of the first character after the opening brace. - *next_idx_ptr is a return-by-reference index to the first character after - the closing brace. - - Returns a new PyList - */ - PY2_UNUSED int kind = PyUnicode_KIND(pystr); - void *str = PyUnicode_DATA(pystr); - Py_ssize_t end_idx = PyUnicode_GET_LENGTH(pystr) - 1; - PyObject *val = NULL; - PyObject *rval = PyList_New(0); - Py_ssize_t next_idx; - if (rval == NULL) - return NULL; - - /* skip whitespace after [ */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - - /* only loop if the array is non-empty */ - if (idx <= end_idx && PyUnicode_READ(kind, str, idx) != ']') { - int trailing_delimiter = 0; - while (idx <= end_idx) { - trailing_delimiter = 0; - /* read any JSON term */ - val = scan_once_unicode(s, pystr, idx, &next_idx); - if (val == NULL) { - goto bail; - } - - if (PyList_Append(rval, val) == -1) - goto bail; - - Py_CLEAR(val); - idx = next_idx; - - /* skip whitespace between term and , */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - - /* bail if the array is closed or we didn't get the , delimiter */ - if (idx > end_idx) break; - if (PyUnicode_READ(kind, str, idx) == ']') { - break; - } - else if (PyUnicode_READ(kind, str, idx) != ',') { - raise_errmsg(ERR_ARRAY_DELIMITER, pystr, idx); - goto bail; - } - idx++; - - /* skip whitespace after , */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - trailing_delimiter = 1; - } - if (trailing_delimiter) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx); - goto bail; - } - } - - /* verify that idx < end_idx, str[idx] should be ']' */ - if (idx > end_idx || PyUnicode_READ(kind, str, idx) != ']') { - if (PyList_GET_SIZE(rval)) { - raise_errmsg(ERR_ARRAY_DELIMITER, pystr, idx); - } else { - raise_errmsg(ERR_ARRAY_VALUE_FIRST, pystr, idx); - } - goto bail; - } - *next_idx_ptr = idx + 1; - return rval; -bail: - Py_XDECREF(val); - Py_DECREF(rval); - return NULL; -} - -static PyObject * -_parse_constant(PyScannerObject *s, PyObject *pystr, PyObject *constant, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON constant from PyString pystr. - constant is the Python string that was found - ("NaN", "Infinity", "-Infinity"). - idx is the index of the first character of the constant - *next_idx_ptr is a return-by-reference index to the first character after - the constant. - - Returns the result of parse_constant - */ - PyObject *rval; - if (s->parse_constant == Py_None) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx); - return NULL; - } - - /* rval = parse_constant(constant) */ - rval = PyObject_CallOneArg(s->parse_constant, constant); - idx += PyString_GET_SIZE(constant); - *next_idx_ptr = idx; - return rval; -} - -#if PY_MAJOR_VERSION < 3 -static PyObject * -_match_number_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON number from PyString pystr. - idx is the index of the first character of the number - *next_idx_ptr is a return-by-reference index to the first character after - the number. - - Returns a new PyObject representation of that number: - PyInt, PyLong, or PyFloat. - May return other types if parse_int or parse_float are set - */ - char *str = PyString_AS_STRING(pystr); - Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1; - Py_ssize_t idx = start; - int is_float = 0; - PyObject *rval; - PyObject *numstr; - - /* read a sign if it's there, make sure it's not the end of the string */ - if (str[idx] == '-') { - if (idx >= end_idx) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, start); - return NULL; - } - idx++; - } - - /* read as many integer digits as we find as long as it doesn't start with 0 */ - if (str[idx] >= '1' && str[idx] <= '9') { - idx++; - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; - } - /* if it starts with 0 we only expect one integer digit */ - else if (str[idx] == '0') { - idx++; - } - /* no integer digits, error */ - else { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, start); - return NULL; - } - - /* if the next char is '.' followed by a digit then read all float digits */ - if (idx < end_idx && str[idx] == '.' && str[idx + 1] >= '0' && str[idx + 1] <= '9') { - is_float = 1; - idx += 2; - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; - } - - /* if the next char is 'e' or 'E' then maybe read the exponent (or backtrack) */ - if (idx < end_idx && (str[idx] == 'e' || str[idx] == 'E')) { - - /* save the index of the 'e' or 'E' just in case we need to backtrack */ - Py_ssize_t e_start = idx; - idx++; - - /* read an exponent sign if present */ - if (idx < end_idx && (str[idx] == '-' || str[idx] == '+')) idx++; - - /* read all digits */ - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; - - /* if we got a digit, then parse as float. if not, backtrack */ - if (str[idx - 1] >= '0' && str[idx - 1] <= '9') { - is_float = 1; - } - else { - idx = e_start; - } - } - - /* copy the section we determined to be a number */ - numstr = PyString_FromStringAndSize(&str[start], idx - start); - if (numstr == NULL) - return NULL; - if (is_float) { - /* parse as a float using a fast path if available, otherwise call user defined method */ - if (s->parse_float != (PyObject *)&PyFloat_Type) { - rval = PyObject_CallOneArg(s->parse_float, numstr); - } - else { - /* rval = PyFloat_FromDouble(PyOS_ascii_atof(PyString_AS_STRING(numstr))); */ - double d = PyOS_string_to_double(PyString_AS_STRING(numstr), - NULL, NULL); - if (d == -1.0 && PyErr_Occurred()) { - Py_DECREF(numstr); - return NULL; - } - rval = PyFloat_FromDouble(d); - } - } - else { - /* parse as an int using a fast path if available, otherwise call user defined method */ - if (s->parse_int != (PyObject *)&PyInt_Type) { - rval = PyObject_CallOneArg(s->parse_int, numstr); - } - else { - rval = PyInt_FromString(PyString_AS_STRING(numstr), NULL, 10); - } - } - Py_DECREF(numstr); - *next_idx_ptr = idx; - return rval; -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -_match_number_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON number from PyUnicode pystr. - idx is the index of the first character of the number - *next_idx_ptr is a return-by-reference index to the first character after - the number. - - Returns a new PyObject representation of that number: - PyInt, PyLong, or PyFloat. - May return other types if parse_int or parse_float are set - */ - PY2_UNUSED int kind = PyUnicode_KIND(pystr); - void *str = PyUnicode_DATA(pystr); - Py_ssize_t end_idx = PyUnicode_GET_LENGTH(pystr) - 1; - Py_ssize_t idx = start; - int is_float = 0; - JSON_UNICHR c; - PyObject *rval; - PyObject *numstr; - - /* read a sign if it's there, make sure it's not the end of the string */ - if (PyUnicode_READ(kind, str, idx) == '-') { - if (idx >= end_idx) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, start); - return NULL; - } - idx++; - } - - /* read as many integer digits as we find as long as it doesn't start with 0 */ - c = PyUnicode_READ(kind, str, idx); - if (c == '0') { - /* if it starts with 0 we only expect one integer digit */ - idx++; - } - else if (IS_DIGIT(c)) { - idx++; - while (idx <= end_idx && IS_DIGIT(PyUnicode_READ(kind, str, idx))) { - idx++; - } - } - else { - /* no integer digits, error */ - raise_errmsg(ERR_EXPECTING_VALUE, pystr, start); - return NULL; - } - - /* if the next char is '.' followed by a digit then read all float digits */ - if (idx < end_idx && - PyUnicode_READ(kind, str, idx) == '.' && - IS_DIGIT(PyUnicode_READ(kind, str, idx + 1))) { - is_float = 1; - idx += 2; - while (idx <= end_idx && IS_DIGIT(PyUnicode_READ(kind, str, idx))) idx++; - } - - /* if the next char is 'e' or 'E' then maybe read the exponent (or backtrack) */ - if (idx < end_idx && - (PyUnicode_READ(kind, str, idx) == 'e' || - PyUnicode_READ(kind, str, idx) == 'E')) { - Py_ssize_t e_start = idx; - idx++; - - /* read an exponent sign if present */ - if (idx < end_idx && - (PyUnicode_READ(kind, str, idx) == '-' || - PyUnicode_READ(kind, str, idx) == '+')) idx++; - - /* read all digits */ - while (idx <= end_idx && IS_DIGIT(PyUnicode_READ(kind, str, idx))) idx++; - - /* if we got a digit, then parse as float. if not, backtrack */ - if (IS_DIGIT(PyUnicode_READ(kind, str, idx - 1))) { - is_float = 1; - } - else { - idx = e_start; - } - } - - /* copy the section we determined to be a number */ -#if PY_MAJOR_VERSION >= 3 - numstr = PyUnicode_Substring(pystr, start, idx); -#else - numstr = PyUnicode_FromUnicode(&((Py_UNICODE *)str)[start], idx - start); -#endif - if (numstr == NULL) - return NULL; - if (is_float) { - /* parse as a float using a fast path if available, otherwise call user defined method */ - if (s->parse_float != (PyObject *)&PyFloat_Type) { - rval = PyObject_CallOneArg(s->parse_float, numstr); - } - else { -#if PY_MAJOR_VERSION >= 3 - rval = PyFloat_FromString(numstr); -#else - rval = PyFloat_FromString(numstr, NULL); -#endif - } - } - else { - /* no fast path for unicode -> int, just call */ - rval = PyObject_CallOneArg(s->parse_int, numstr); - } - Py_DECREF(numstr); - *next_idx_ptr = idx; - return rval; -} - -#if PY_MAJOR_VERSION < 3 -static PyObject * -scan_once_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read one JSON term (of any kind) from PyString pystr. - idx is the index of the first character of the term - *next_idx_ptr is a return-by-reference index to the first character after - the number. - - Returns a new PyObject representation of the term. - */ - char *str = PyString_AS_STRING(pystr); - Py_ssize_t length = PyString_GET_SIZE(pystr); - PyObject *rval = NULL; - int fallthrough = 0; - if (idx < 0 || idx >= length) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx); - return NULL; - } - switch (str[idx]) { - case '"': - /* string */ - rval = scanstring_str(pystr, idx + 1, - PyString_AS_STRING(s->encoding), - s->strict, - next_idx_ptr); - break; - case '{': - /* object */ - if (Py_EnterRecursiveCall(" while decoding a JSON object " - "from a string")) - return NULL; - rval = _parse_object_str(s, pystr, idx + 1, next_idx_ptr); - Py_LeaveRecursiveCall(); - break; - case '[': - /* array */ - if (Py_EnterRecursiveCall(" while decoding a JSON array " - "from a string")) - return NULL; - rval = _parse_array_str(s, pystr, idx + 1, next_idx_ptr); - Py_LeaveRecursiveCall(); - break; - case 'n': - /* null */ - if ((idx + 3 < length) && str[idx + 1] == 'u' && str[idx + 2] == 'l' && str[idx + 3] == 'l') { - Py_INCREF(Py_None); - *next_idx_ptr = idx + 4; - rval = Py_None; - } - else - fallthrough = 1; - break; - case 't': - /* true */ - if ((idx + 3 < length) && str[idx + 1] == 'r' && str[idx + 2] == 'u' && str[idx + 3] == 'e') { - Py_INCREF(Py_True); - *next_idx_ptr = idx + 4; - rval = Py_True; - } - else - fallthrough = 1; - break; - case 'f': - /* false */ - if ((idx + 4 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'l' && str[idx + 3] == 's' && str[idx + 4] == 'e') { - Py_INCREF(Py_False); - *next_idx_ptr = idx + 5; - rval = Py_False; - } - else - fallthrough = 1; - break; - case 'N': - /* NaN */ - if ((idx + 2 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'N') { - rval = _parse_constant(s, pystr, JSON_NaN, idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - case 'I': - /* Infinity */ - if ((idx + 7 < length) && str[idx + 1] == 'n' && str[idx + 2] == 'f' && str[idx + 3] == 'i' && str[idx + 4] == 'n' && str[idx + 5] == 'i' && str[idx + 6] == 't' && str[idx + 7] == 'y') { - rval = _parse_constant(s, pystr, JSON_Infinity, idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - case '-': - /* -Infinity */ - if ((idx + 8 < length) && str[idx + 1] == 'I' && str[idx + 2] == 'n' && str[idx + 3] == 'f' && str[idx + 4] == 'i' && str[idx + 5] == 'n' && str[idx + 6] == 'i' && str[idx + 7] == 't' && str[idx + 8] == 'y') { - rval = _parse_constant(s, pystr, JSON_NegInfinity, idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - default: - fallthrough = 1; - } - /* Didn't find a string, object, array, or named constant. Look for a number. */ - if (fallthrough) - rval = _match_number_str(s, pystr, idx, next_idx_ptr); - return rval; -} -#endif /* PY_MAJOR_VERSION < 3 */ - - -static PyObject * -scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read one JSON term (of any kind) from PyUnicode pystr. - idx is the index of the first character of the term - *next_idx_ptr is a return-by-reference index to the first character after - the number. - - Returns a new PyObject representation of the term. - */ - PY2_UNUSED int kind = PyUnicode_KIND(pystr); - void *str = PyUnicode_DATA(pystr); - Py_ssize_t length = PyUnicode_GET_LENGTH(pystr); - PyObject *rval = NULL; - int fallthrough = 0; - if (idx < 0 || idx >= length) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx); - return NULL; - } - switch (PyUnicode_READ(kind, str, idx)) { - case '"': - /* string */ - rval = scanstring_unicode(pystr, idx + 1, - s->strict, - next_idx_ptr); - break; - case '{': - /* object */ - if (Py_EnterRecursiveCall(" while decoding a JSON object " - "from a unicode string")) - return NULL; - rval = _parse_object_unicode(s, pystr, idx + 1, next_idx_ptr); - Py_LeaveRecursiveCall(); - break; - case '[': - /* array */ - if (Py_EnterRecursiveCall(" while decoding a JSON array " - "from a unicode string")) - return NULL; - rval = _parse_array_unicode(s, pystr, idx + 1, next_idx_ptr); - Py_LeaveRecursiveCall(); - break; - case 'n': - /* null */ - if ((idx + 3 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'u' && - PyUnicode_READ(kind, str, idx + 2) == 'l' && - PyUnicode_READ(kind, str, idx + 3) == 'l') { - Py_INCREF(Py_None); - *next_idx_ptr = idx + 4; - rval = Py_None; - } - else - fallthrough = 1; - break; - case 't': - /* true */ - if ((idx + 3 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'r' && - PyUnicode_READ(kind, str, idx + 2) == 'u' && - PyUnicode_READ(kind, str, idx + 3) == 'e') { - Py_INCREF(Py_True); - *next_idx_ptr = idx + 4; - rval = Py_True; - } - else - fallthrough = 1; - break; - case 'f': - /* false */ - if ((idx + 4 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'a' && - PyUnicode_READ(kind, str, idx + 2) == 'l' && - PyUnicode_READ(kind, str, idx + 3) == 's' && - PyUnicode_READ(kind, str, idx + 4) == 'e') { - Py_INCREF(Py_False); - *next_idx_ptr = idx + 5; - rval = Py_False; - } - else - fallthrough = 1; - break; - case 'N': - /* NaN */ - if ((idx + 2 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'a' && - PyUnicode_READ(kind, str, idx + 2) == 'N') { - rval = _parse_constant(s, pystr, JSON_NaN, idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - case 'I': - /* Infinity */ - if ((idx + 7 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'n' && - PyUnicode_READ(kind, str, idx + 2) == 'f' && - PyUnicode_READ(kind, str, idx + 3) == 'i' && - PyUnicode_READ(kind, str, idx + 4) == 'n' && - PyUnicode_READ(kind, str, idx + 5) == 'i' && - PyUnicode_READ(kind, str, idx + 6) == 't' && - PyUnicode_READ(kind, str, idx + 7) == 'y') { - rval = _parse_constant(s, pystr, JSON_Infinity, idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - case '-': - /* -Infinity */ - if ((idx + 8 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'I' && - PyUnicode_READ(kind, str, idx + 2) == 'n' && - PyUnicode_READ(kind, str, idx + 3) == 'f' && - PyUnicode_READ(kind, str, idx + 4) == 'i' && - PyUnicode_READ(kind, str, idx + 5) == 'n' && - PyUnicode_READ(kind, str, idx + 6) == 'i' && - PyUnicode_READ(kind, str, idx + 7) == 't' && - PyUnicode_READ(kind, str, idx + 8) == 'y') { - rval = _parse_constant(s, pystr, JSON_NegInfinity, idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - default: - fallthrough = 1; - } - /* Didn't find a string, object, array, or named constant. Look for a number. */ - if (fallthrough) - rval = _match_number_unicode(s, pystr, idx, next_idx_ptr); - return rval; -} - -static PyObject * -scanner_call(PyObject *self, PyObject *args, PyObject *kwds) -{ - /* Python callable interface to scan_once_{str,unicode} */ - PyObject *pystr; - PyObject *rval; - Py_ssize_t idx; - Py_ssize_t next_idx = -1; - static char *kwlist[] = {"string", "idx", NULL}; - PyScannerObject *s; - assert(PyScanner_Check(self)); - s = (PyScannerObject *)self; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&:scan_once", kwlist, &pystr, _convertPyInt_AsSsize_t, &idx)) - return NULL; - - if (PyUnicode_Check(pystr)) { - if (PyUnicode_READY(pystr)) - return NULL; - rval = scan_once_unicode(s, pystr, idx, &next_idx); - } -#if PY_MAJOR_VERSION < 3 - else if (PyString_Check(pystr)) { - rval = scan_once_str(s, pystr, idx, &next_idx); - } -#endif /* PY_MAJOR_VERSION < 3 */ - else { - PyErr_Format(PyExc_TypeError, - "first argument must be a string, not %.80s", - Py_TYPE(pystr)->tp_name); - return NULL; - } - PyDict_Clear(s->memo); - return _build_rval_index_tuple(rval, next_idx); -} - -static PyObject * -JSON_ParseEncoding(PyObject *encoding) -{ - if (encoding == Py_None) - return JSON_InternFromString(DEFAULT_ENCODING); -#if PY_MAJOR_VERSION >= 3 - if (PyUnicode_Check(encoding)) { - if (PyUnicode_AsUTF8(encoding) == NULL) { - return NULL; - } - Py_INCREF(encoding); - return encoding; - } -#else /* PY_MAJOR_VERSION >= 3 */ - if (PyString_Check(encoding)) { - Py_INCREF(encoding); - return encoding; - } - if (PyUnicode_Check(encoding)) - return PyUnicode_AsEncodedString(encoding, NULL, NULL); -#endif /* PY_MAJOR_VERSION >= 3 */ - PyErr_SetString(PyExc_TypeError, "encoding must be a string"); - return NULL; -} - -static PyObject * -scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - /* Initialize Scanner object */ - PyObject *ctx; - static char *kwlist[] = {"context", NULL}; - PyScannerObject *s; - PyObject *encoding; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:make_scanner", kwlist, &ctx)) - return NULL; - - s = (PyScannerObject *)type->tp_alloc(type, 0); - if (s == NULL) - return NULL; - - if (s->memo == NULL) { - s->memo = PyDict_New(); - if (s->memo == NULL) - goto bail; - } - - encoding = PyObject_GetAttrString(ctx, "encoding"); - if (encoding == NULL) - goto bail; - s->encoding = JSON_ParseEncoding(encoding); - Py_XDECREF(encoding); - if (s->encoding == NULL) - goto bail; - - /* All of these will fail "gracefully" so we don't need to verify them */ - s->strict_bool = PyObject_GetAttrString(ctx, "strict"); - if (s->strict_bool == NULL) - goto bail; - s->strict = PyObject_IsTrue(s->strict_bool); - if (s->strict < 0) - goto bail; - s->object_hook = PyObject_GetAttrString(ctx, "object_hook"); - if (s->object_hook == NULL) - goto bail; - s->pairs_hook = PyObject_GetAttrString(ctx, "object_pairs_hook"); - if (s->pairs_hook == NULL) - goto bail; - s->parse_float = PyObject_GetAttrString(ctx, "parse_float"); - if (s->parse_float == NULL) - goto bail; - s->parse_int = PyObject_GetAttrString(ctx, "parse_int"); - if (s->parse_int == NULL) - goto bail; - s->parse_constant = PyObject_GetAttrString(ctx, "parse_constant"); - if (s->parse_constant == NULL) - goto bail; - - return (PyObject *)s; - -bail: - Py_DECREF(s); - return NULL; -} - -PyDoc_STRVAR(scanner_doc, "JSON scanner object"); - -static -PyTypeObject PyScannerType = { - PyVarObject_HEAD_INIT(NULL, 0) - "simplejson._speedups.Scanner", /* tp_name */ - sizeof(PyScannerObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - scanner_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - scanner_call, /* tp_call */ - 0, /* tp_str */ - 0,/* PyObject_GenericGetAttr, */ /* tp_getattro */ - 0,/* PyObject_GenericSetAttr, */ /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - scanner_doc, /* tp_doc */ - scanner_traverse, /* tp_traverse */ - scanner_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - scanner_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0,/* PyType_GenericAlloc, */ /* tp_alloc */ - scanner_new, /* tp_new */ - 0,/* PyObject_GC_Del, */ /* tp_free */ -}; - -static PyObject * -encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = { - "markers", - "default", - "encoder", - "indent", - "key_separator", - "item_separator", - "sort_keys", - "skipkeys", - "allow_nan", - "key_memo", - "use_decimal", - "namedtuple_as_object", - "tuple_as_array", - "int_as_string_bitcount", - "item_sort_key", - "encoding", - "for_json", - "ignore_nan", - "Decimal", - "iterable_as_array", - NULL}; - - PyEncoderObject *s; - PyObject *markers, *defaultfn, *encoder, *indent, *key_separator; - PyObject *item_separator, *sort_keys, *skipkeys, *allow_nan, *key_memo; - PyObject *use_decimal, *namedtuple_as_object, *tuple_as_array, *iterable_as_array; - PyObject *int_as_string_bitcount, *item_sort_key, *encoding, *for_json; - PyObject *ignore_nan, *Decimal; - int is_true; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOOOOOOOOOOOOOOOOOOO:make_encoder", kwlist, - &markers, &defaultfn, &encoder, &indent, &key_separator, &item_separator, - &sort_keys, &skipkeys, &allow_nan, &key_memo, &use_decimal, - &namedtuple_as_object, &tuple_as_array, - &int_as_string_bitcount, &item_sort_key, &encoding, &for_json, - &ignore_nan, &Decimal, &iterable_as_array)) - return NULL; - - s = (PyEncoderObject *)type->tp_alloc(type, 0); - if (s == NULL) - return NULL; - - Py_INCREF(markers); - s->markers = markers; - Py_INCREF(defaultfn); - s->defaultfn = defaultfn; - Py_INCREF(encoder); - s->encoder = encoder; -#if PY_MAJOR_VERSION >= 3 - if (encoding == Py_None) { - s->encoding = NULL; - } - else -#endif /* PY_MAJOR_VERSION >= 3 */ - { - s->encoding = JSON_ParseEncoding(encoding); - if (s->encoding == NULL) - goto bail; - } - Py_INCREF(indent); - s->indent = indent; - Py_INCREF(key_separator); - s->key_separator = key_separator; - Py_INCREF(item_separator); - s->item_separator = item_separator; - Py_INCREF(skipkeys); - s->skipkeys_bool = skipkeys; - s->skipkeys = PyObject_IsTrue(skipkeys); - if (s->skipkeys < 0) - goto bail; - Py_INCREF(key_memo); - s->key_memo = key_memo; - s->fast_encode = (PyCFunction_Check(s->encoder) && PyCFunction_GetFunction(s->encoder) == (PyCFunction)py_encode_basestring_ascii); - is_true = PyObject_IsTrue(ignore_nan); - if (is_true < 0) - goto bail; - s->allow_or_ignore_nan = is_true ? JSON_IGNORE_NAN : 0; - is_true = PyObject_IsTrue(allow_nan); - if (is_true < 0) - goto bail; - s->allow_or_ignore_nan |= is_true ? JSON_ALLOW_NAN : 0; - s->use_decimal = PyObject_IsTrue(use_decimal); - if (s->use_decimal < 0) - goto bail; - s->namedtuple_as_object = PyObject_IsTrue(namedtuple_as_object); - if (s->namedtuple_as_object < 0) - goto bail; - s->tuple_as_array = PyObject_IsTrue(tuple_as_array); - if (s->tuple_as_array < 0) - goto bail; - s->iterable_as_array = PyObject_IsTrue(iterable_as_array); - if (s->iterable_as_array < 0) - goto bail; - if (PyInt_Check(int_as_string_bitcount) || PyLong_Check(int_as_string_bitcount)) { - static const unsigned long long_long_bitsize = SIZEOF_LONG_LONG * 8; - long int_as_string_bitcount_val = PyLong_AsLong(int_as_string_bitcount); - if (int_as_string_bitcount_val > 0 && int_as_string_bitcount_val < (long)long_long_bitsize) { - s->max_long_size = PyLong_FromUnsignedLongLong(1ULL << (int)int_as_string_bitcount_val); - s->min_long_size = PyLong_FromLongLong(-1LL << (int)int_as_string_bitcount_val); - if (s->min_long_size == NULL || s->max_long_size == NULL) { - goto bail; - } - } - else { - PyErr_Format(PyExc_TypeError, - "int_as_string_bitcount (%ld) must be greater than 0 and less than the number of bits of a `long long` type (%lu bits)", - int_as_string_bitcount_val, long_long_bitsize); - goto bail; - } - } - else if (int_as_string_bitcount == Py_None) { - Py_INCREF(Py_None); - s->max_long_size = Py_None; - Py_INCREF(Py_None); - s->min_long_size = Py_None; - } - else { - PyErr_SetString(PyExc_TypeError, "int_as_string_bitcount must be None or an integer"); - goto bail; - } - if (item_sort_key != Py_None) { - if (!PyCallable_Check(item_sort_key)) { - PyErr_SetString(PyExc_TypeError, "item_sort_key must be None or callable"); - goto bail; - } - } - else { - is_true = PyObject_IsTrue(sort_keys); - if (is_true < 0) - goto bail; - if (is_true) { - static PyObject *itemgetter0 = NULL; - if (!itemgetter0) { - PyObject *operator = PyImport_ImportModule("operator"); - if (!operator) - goto bail; - itemgetter0 = PyObject_CallMethod(operator, "itemgetter", "i", 0); - Py_DECREF(operator); - } - item_sort_key = itemgetter0; - if (!item_sort_key) - goto bail; - } - } - if (item_sort_key == Py_None) { - Py_INCREF(Py_None); - s->item_sort_kw = Py_None; - } - else { - s->item_sort_kw = PyDict_New(); - if (s->item_sort_kw == NULL) - goto bail; - if (PyDict_SetItemString(s->item_sort_kw, "key", item_sort_key)) - goto bail; - } - Py_INCREF(sort_keys); - s->sort_keys = sort_keys; - Py_INCREF(item_sort_key); - s->item_sort_key = item_sort_key; - Py_INCREF(Decimal); - s->Decimal = Decimal; - s->for_json = PyObject_IsTrue(for_json); - if (s->for_json < 0) - goto bail; - - return (PyObject *)s; - -bail: - Py_DECREF(s); - return NULL; -} - -static PyObject * -encoder_call(PyObject *self, PyObject *args, PyObject *kwds) -{ - /* Python callable interface to encode_listencode_obj */ - static char *kwlist[] = {"obj", "_current_indent_level", NULL}; - PyObject *obj; - Py_ssize_t indent_level; - PyEncoderObject *s; - JSON_Accu rval; - assert(PyEncoder_Check(self)); - s = (PyEncoderObject *)self; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&:_iterencode", kwlist, - &obj, _convertPyInt_AsSsize_t, &indent_level)) - return NULL; - if (JSON_Accu_Init(&rval)) - return NULL; - if (encoder_listencode_obj(s, &rval, obj, indent_level)) { - JSON_Accu_Destroy(&rval); - return NULL; - } - return JSON_Accu_FinishAsList(&rval); -} - -static PyObject * -_encoded_const(PyObject *obj) -{ - /* Return the JSON string representation of None, True, False */ - if (obj == Py_None) { - static PyObject *s_null = NULL; - if (s_null == NULL) { - s_null = JSON_InternFromString("null"); - } - Py_INCREF(s_null); - return s_null; - } - else if (obj == Py_True) { - static PyObject *s_true = NULL; - if (s_true == NULL) { - s_true = JSON_InternFromString("true"); - } - Py_INCREF(s_true); - return s_true; - } - else if (obj == Py_False) { - static PyObject *s_false = NULL; - if (s_false == NULL) { - s_false = JSON_InternFromString("false"); - } - Py_INCREF(s_false); - return s_false; - } - else { - PyErr_SetString(PyExc_ValueError, "not a const"); - return NULL; - } -} - -static PyObject * -encoder_encode_float(PyEncoderObject *s, PyObject *obj) -{ - /* Return the JSON representation of a PyFloat */ - double i = PyFloat_AS_DOUBLE(obj); - if (!Py_IS_FINITE(i)) { - if (!s->allow_or_ignore_nan) { - PyErr_SetString(PyExc_ValueError, "Out of range float values are not JSON compliant"); - return NULL; - } - if (s->allow_or_ignore_nan & JSON_IGNORE_NAN) { - return _encoded_const(Py_None); - } - /* JSON_ALLOW_NAN is set */ - else if (i > 0) { - Py_INCREF(JSON_Infinity); - return JSON_Infinity; - } - else if (i < 0) { - Py_INCREF(JSON_NegInfinity); - return JSON_NegInfinity; - } - else { - Py_INCREF(JSON_NaN); - return JSON_NaN; - } - } - /* Use a better float format here? */ - if (PyFloat_CheckExact(obj)) { - return PyObject_Repr(obj); - } - else { - /* See #118, do not trust custom str/repr */ - PyObject *res; - PyObject *tmp = PyObject_CallOneArg((PyObject *)&PyFloat_Type, obj); - if (tmp == NULL) { - return NULL; - } - res = PyObject_Repr(tmp); - Py_DECREF(tmp); - return res; - } -} - -static PyObject * -encoder_encode_string(PyEncoderObject *s, PyObject *obj) -{ - /* Return the JSON representation of a string */ - PyObject *encoded; - - if (s->fast_encode) { - return py_encode_basestring_ascii(NULL, obj); - } - encoded = PyObject_CallOneArg(s->encoder, obj); - if (encoded != NULL && -#if PY_MAJOR_VERSION < 3 - !PyString_Check(encoded) && -#endif /* PY_MAJOR_VERSION < 3 */ - !PyUnicode_Check(encoded)) - { - PyErr_Format(PyExc_TypeError, - "encoder() must return a string, not %.80s", - Py_TYPE(encoded)->tp_name); - Py_DECREF(encoded); - return NULL; - } - return encoded; -} - -static int -_steal_accumulate(JSON_Accu *accu, PyObject *stolen) -{ - /* Append stolen and then decrement its reference count */ - int rval = JSON_Accu_Accumulate(accu, stolen); - Py_DECREF(stolen); - return rval; -} - -static int -encoder_listencode_obj(PyEncoderObject *s, JSON_Accu *rval, PyObject *obj, Py_ssize_t indent_level) -{ - /* Encode Python object obj to a JSON term, rval is a PyList */ - int rv = -1; - do { - PyObject *newobj; - if (obj == Py_None || obj == Py_True || obj == Py_False) { - PyObject *cstr = _encoded_const(obj); - if (cstr != NULL) - rv = _steal_accumulate(rval, cstr); - } - else if ((PyBytes_Check(obj) && s->encoding != NULL) || - PyUnicode_Check(obj)) - { - PyObject *encoded = encoder_encode_string(s, obj); - if (encoded != NULL) - rv = _steal_accumulate(rval, encoded); - } - else if (PyInt_Check(obj) || PyLong_Check(obj)) { - PyObject *encoded; - if (PyInt_CheckExact(obj) || PyLong_CheckExact(obj)) { - encoded = PyObject_Str(obj); - } - else { - /* See #118, do not trust custom str/repr */ - PyObject *tmp = PyObject_CallOneArg((PyObject *)&PyLong_Type, obj); - if (tmp == NULL) { - encoded = NULL; - } - else { - encoded = PyObject_Str(tmp); - Py_DECREF(tmp); - } - } - if (encoded != NULL) { - encoded = maybe_quote_bigint(s, encoded, obj); - if (encoded == NULL) - break; - rv = _steal_accumulate(rval, encoded); - } - } - else if (PyFloat_Check(obj)) { - PyObject *encoded = encoder_encode_float(s, obj); - if (encoded != NULL) - rv = _steal_accumulate(rval, encoded); - } - else if (s->for_json && _call_json_method(obj, FOR_JSON_METHOD_NAME, &newobj)) { - if (newobj == NULL) { - return -1; - } - if (Py_EnterRecursiveCall(" while encoding a JSON object")) { - Py_DECREF(newobj); - return rv; - } - rv = encoder_listencode_obj(s, rval, newobj, indent_level); - Py_DECREF(newobj); - Py_LeaveRecursiveCall(); - } - else if (s->namedtuple_as_object && _call_json_method(obj, ASDICT_METHOD_NAME, &newobj)) { - if (newobj == NULL) { - return -1; - } - if (Py_EnterRecursiveCall(" while encoding a JSON object")) { - Py_DECREF(newobj); - return rv; - } - if (PyDict_Check(newobj)) { - rv = encoder_listencode_dict(s, rval, newobj, indent_level); - } else { - PyErr_Format( - PyExc_TypeError, - "_asdict() must return a dict, not %.80s", - Py_TYPE(newobj)->tp_name - ); - rv = -1; - } - Py_DECREF(newobj); - Py_LeaveRecursiveCall(); - } - else if (PyList_Check(obj) || (s->tuple_as_array && PyTuple_Check(obj))) { - if (Py_EnterRecursiveCall(" while encoding a JSON object")) - return rv; - rv = encoder_listencode_list(s, rval, obj, indent_level); - Py_LeaveRecursiveCall(); - } - else if (PyDict_Check(obj)) { - if (Py_EnterRecursiveCall(" while encoding a JSON object")) - return rv; - rv = encoder_listencode_dict(s, rval, obj, indent_level); - Py_LeaveRecursiveCall(); - } - else if (s->use_decimal && PyObject_TypeCheck(obj, (PyTypeObject *)s->Decimal)) { - PyObject *encoded = PyObject_Str(obj); - if (encoded != NULL) - rv = _steal_accumulate(rval, encoded); - } - else if (is_raw_json(obj)) - { - PyObject *encoded = PyObject_GetAttrString(obj, "encoded_json"); - if (encoded != NULL) - rv = _steal_accumulate(rval, encoded); - } - else { - PyObject *ident = NULL; - PyObject *newobj; - if (s->iterable_as_array) { - newobj = PyObject_GetIter(obj); - if (newobj == NULL) - PyErr_Clear(); - else { - rv = encoder_listencode_list(s, rval, newobj, indent_level); - Py_DECREF(newobj); - break; - } - } - if (s->markers != Py_None) { - int has_key; - ident = PyLong_FromVoidPtr(obj); - if (ident == NULL) - break; - has_key = PyDict_Contains(s->markers, ident); - if (has_key) { - if (has_key != -1) - PyErr_SetString(PyExc_ValueError, "Circular reference detected"); - Py_DECREF(ident); - break; - } - if (PyDict_SetItem(s->markers, ident, obj)) { - Py_DECREF(ident); - break; - } - } - if (Py_EnterRecursiveCall(" while encoding a JSON object")) - return rv; - newobj = PyObject_CallOneArg(s->defaultfn, obj); - if (newobj == NULL) { - Py_XDECREF(ident); - Py_LeaveRecursiveCall(); - break; - } - rv = encoder_listencode_obj(s, rval, newobj, indent_level); - Py_LeaveRecursiveCall(); - Py_DECREF(newobj); - if (rv) { - Py_XDECREF(ident); - rv = -1; - } - else if (ident != NULL) { - if (PyDict_DelItem(s->markers, ident)) { - Py_XDECREF(ident); - rv = -1; - } - Py_XDECREF(ident); - } - } - } while (0); - return rv; -} - -static int -encoder_listencode_dict(PyEncoderObject *s, JSON_Accu *rval, PyObject *dct, Py_ssize_t indent_level) -{ - /* Encode Python dict dct a JSON term */ - static PyObject *open_dict = NULL; - static PyObject *close_dict = NULL; - static PyObject *empty_dict = NULL; - PyObject *kstr = NULL; - PyObject *ident = NULL; - PyObject *iter = NULL; - PyObject *item = NULL; - PyObject *items = NULL; - PyObject *encoded = NULL; - Py_ssize_t idx; - - if (open_dict == NULL || close_dict == NULL || empty_dict == NULL) { - open_dict = JSON_InternFromString("{"); - close_dict = JSON_InternFromString("}"); - empty_dict = JSON_InternFromString("{}"); - if (open_dict == NULL || close_dict == NULL || empty_dict == NULL) - return -1; - } - if (PyDict_Size(dct) == 0) - return JSON_Accu_Accumulate(rval, empty_dict); - - if (s->markers != Py_None) { - int has_key; - ident = PyLong_FromVoidPtr(dct); - if (ident == NULL) - goto bail; - has_key = PyDict_Contains(s->markers, ident); - if (has_key) { - if (has_key != -1) - PyErr_SetString(PyExc_ValueError, "Circular reference detected"); - goto bail; - } - if (PyDict_SetItem(s->markers, ident, dct)) { - goto bail; - } - } - - if (JSON_Accu_Accumulate(rval, open_dict)) - goto bail; - - if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level += 1; - /* - newline_indent = '\n' + (_indent * _current_indent_level) - separator = _item_separator + newline_indent - buf += newline_indent - */ - } - - iter = encoder_dict_iteritems(s, dct); - if (iter == NULL) - goto bail; - - idx = 0; - while ((item = PyIter_Next(iter))) { - PyObject *encoded, *key, *value; - if (!PyTuple_Check(item) || Py_SIZE(item) != 2) { - PyErr_SetString(PyExc_ValueError, "items must return 2-tuples"); - goto bail; - } - key = PyTuple_GET_ITEM(item, 0); - if (key == NULL) - goto bail; - value = PyTuple_GET_ITEM(item, 1); - if (value == NULL) - goto bail; - - kstr = encoder_stringify_key(s, key); - if (kstr == NULL) - goto bail; - else if (kstr == Py_None) { - /* skipkeys */ - Py_DECREF(item); - Py_DECREF(kstr); - continue; - } - if (idx) { - if (JSON_Accu_Accumulate(rval, s->item_separator)) - goto bail; - } - /* - * Only cache the encoding of string keys. False and True are - * indistinguishable from 0 and 1 in a dictionary lookup and there - * may be other quirks with user defined subclasses. - */ - encoded = PyDict_GetItem(s->key_memo, kstr); - if (encoded != NULL) { - Py_INCREF(encoded); - Py_CLEAR(kstr); - } else { - encoded = encoder_encode_string(s, kstr); - Py_CLEAR(kstr); - if (encoded == NULL) - goto bail; - if (PyDict_SetItem(s->key_memo, key, encoded)) - goto bail; - } - if (JSON_Accu_Accumulate(rval, encoded)) { - goto bail; - } - Py_CLEAR(encoded); - if (JSON_Accu_Accumulate(rval, s->key_separator)) - goto bail; - if (encoder_listencode_obj(s, rval, value, indent_level)) - goto bail; - Py_CLEAR(item); - idx += 1; - } - Py_CLEAR(iter); - if (PyErr_Occurred()) - goto bail; - if (ident != NULL) { - if (PyDict_DelItem(s->markers, ident)) - goto bail; - Py_CLEAR(ident); - } - if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level -= 1; - /* - yield '\n' + (_indent * _current_indent_level) - */ - } - if (JSON_Accu_Accumulate(rval, close_dict)) - goto bail; - return 0; - -bail: - Py_XDECREF(encoded); - Py_XDECREF(items); - Py_XDECREF(item); - Py_XDECREF(iter); - Py_XDECREF(kstr); - Py_XDECREF(ident); - return -1; -} - - -static int -encoder_listencode_list(PyEncoderObject *s, JSON_Accu *rval, PyObject *seq, Py_ssize_t indent_level) -{ - /* Encode Python list seq to a JSON term */ - static PyObject *open_array = NULL; - static PyObject *close_array = NULL; - static PyObject *empty_array = NULL; - PyObject *ident = NULL; - PyObject *iter = NULL; - PyObject *obj = NULL; - int is_true; - int i = 0; - - if (open_array == NULL || close_array == NULL || empty_array == NULL) { - open_array = JSON_InternFromString("["); - close_array = JSON_InternFromString("]"); - empty_array = JSON_InternFromString("[]"); - if (open_array == NULL || close_array == NULL || empty_array == NULL) - return -1; - } - ident = NULL; - is_true = PyObject_IsTrue(seq); - if (is_true == -1) - return -1; - else if (is_true == 0) - return JSON_Accu_Accumulate(rval, empty_array); - - if (s->markers != Py_None) { - int has_key; - ident = PyLong_FromVoidPtr(seq); - if (ident == NULL) - goto bail; - has_key = PyDict_Contains(s->markers, ident); - if (has_key) { - if (has_key != -1) - PyErr_SetString(PyExc_ValueError, "Circular reference detected"); - goto bail; - } - if (PyDict_SetItem(s->markers, ident, seq)) { - goto bail; - } - } - - iter = PyObject_GetIter(seq); - if (iter == NULL) - goto bail; - - if (JSON_Accu_Accumulate(rval, open_array)) - goto bail; - if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level += 1; - /* - newline_indent = '\n' + (_indent * _current_indent_level) - separator = _item_separator + newline_indent - buf += newline_indent - */ - } - while ((obj = PyIter_Next(iter))) { - if (i) { - if (JSON_Accu_Accumulate(rval, s->item_separator)) - goto bail; - } - if (encoder_listencode_obj(s, rval, obj, indent_level)) - goto bail; - i++; - Py_CLEAR(obj); - } - Py_CLEAR(iter); - if (PyErr_Occurred()) - goto bail; - if (ident != NULL) { - if (PyDict_DelItem(s->markers, ident)) - goto bail; - Py_CLEAR(ident); - } - if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level -= 1; - /* - yield '\n' + (_indent * _current_indent_level) - */ - } - if (JSON_Accu_Accumulate(rval, close_array)) - goto bail; - return 0; - -bail: - Py_XDECREF(obj); - Py_XDECREF(iter); - Py_XDECREF(ident); - return -1; -} - -static void -encoder_dealloc(PyObject *self) -{ - /* bpo-31095: UnTrack is needed before calling any callbacks */ - PyObject_GC_UnTrack(self); - encoder_clear(self); - Py_TYPE(self)->tp_free(self); -} - -static int -encoder_traverse(PyObject *self, visitproc visit, void *arg) -{ - PyEncoderObject *s; - assert(PyEncoder_Check(self)); - s = (PyEncoderObject *)self; - Py_VISIT(s->markers); - Py_VISIT(s->defaultfn); - Py_VISIT(s->encoder); - Py_VISIT(s->encoding); - Py_VISIT(s->indent); - Py_VISIT(s->key_separator); - Py_VISIT(s->item_separator); - Py_VISIT(s->key_memo); - Py_VISIT(s->sort_keys); - Py_VISIT(s->item_sort_kw); - Py_VISIT(s->item_sort_key); - Py_VISIT(s->max_long_size); - Py_VISIT(s->min_long_size); - Py_VISIT(s->Decimal); - return 0; -} - -static int -encoder_clear(PyObject *self) -{ - /* Deallocate Encoder */ - PyEncoderObject *s; - assert(PyEncoder_Check(self)); - s = (PyEncoderObject *)self; - Py_CLEAR(s->markers); - Py_CLEAR(s->defaultfn); - Py_CLEAR(s->encoder); - Py_CLEAR(s->encoding); - Py_CLEAR(s->indent); - Py_CLEAR(s->key_separator); - Py_CLEAR(s->item_separator); - Py_CLEAR(s->key_memo); - Py_CLEAR(s->skipkeys_bool); - Py_CLEAR(s->sort_keys); - Py_CLEAR(s->item_sort_kw); - Py_CLEAR(s->item_sort_key); - Py_CLEAR(s->max_long_size); - Py_CLEAR(s->min_long_size); - Py_CLEAR(s->Decimal); - return 0; -} - -PyDoc_STRVAR(encoder_doc, "_iterencode(obj, _current_indent_level) -> iterable"); - -static -PyTypeObject PyEncoderType = { - PyVarObject_HEAD_INIT(NULL, 0) - "simplejson._speedups.Encoder", /* tp_name */ - sizeof(PyEncoderObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - encoder_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - encoder_call, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - encoder_doc, /* tp_doc */ - encoder_traverse, /* tp_traverse */ - encoder_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - encoder_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - encoder_new, /* tp_new */ - 0, /* tp_free */ -}; - -static PyMethodDef speedups_methods[] = { - {"encode_basestring_ascii", - (PyCFunction)py_encode_basestring_ascii, - METH_O, - pydoc_encode_basestring_ascii}, - {"scanstring", - (PyCFunction)py_scanstring, - METH_VARARGS, - pydoc_scanstring}, - {NULL, NULL, 0, NULL} -}; - -PyDoc_STRVAR(module_doc, -"simplejson speedups\n"); - -#if PY_MAJOR_VERSION >= 3 -static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - "_speedups", /* m_name */ - module_doc, /* m_doc */ - -1, /* m_size */ - speedups_methods, /* m_methods */ - NULL, /* m_reload */ - NULL, /* m_traverse */ - NULL, /* m_clear*/ - NULL, /* m_free */ -}; -#endif - -PyObject * -import_dependency(char *module_name, char *attr_name) -{ - PyObject *rval; - PyObject *module = PyImport_ImportModule(module_name); - if (module == NULL) - return NULL; - rval = PyObject_GetAttrString(module, attr_name); - Py_DECREF(module); - return rval; -} - -static int -init_constants(void) -{ - JSON_NaN = JSON_InternFromString("NaN"); - if (JSON_NaN == NULL) - return 0; - JSON_Infinity = JSON_InternFromString("Infinity"); - if (JSON_Infinity == NULL) - return 0; - JSON_NegInfinity = JSON_InternFromString("-Infinity"); - if (JSON_NegInfinity == NULL) - return 0; -#if PY_MAJOR_VERSION >= 3 - JSON_EmptyUnicode = PyUnicode_New(0, 127); -#else /* PY_MAJOR_VERSION >= 3 */ - JSON_EmptyStr = PyString_FromString(""); - if (JSON_EmptyStr == NULL) - return 0; - JSON_EmptyUnicode = PyUnicode_FromUnicode(NULL, 0); -#endif /* PY_MAJOR_VERSION >= 3 */ - if (JSON_EmptyUnicode == NULL) - return 0; - - return 1; -} - -static PyObject * -moduleinit(void) -{ - PyObject *m; - if (PyType_Ready(&PyScannerType) < 0) - return NULL; - if (PyType_Ready(&PyEncoderType) < 0) - return NULL; - if (!init_constants()) - return NULL; - -#if PY_MAJOR_VERSION >= 3 - m = PyModule_Create(&moduledef); -#else - m = Py_InitModule3("_speedups", speedups_methods, module_doc); -#endif - Py_INCREF((PyObject*)&PyScannerType); - PyModule_AddObject(m, "make_scanner", (PyObject*)&PyScannerType); - Py_INCREF((PyObject*)&PyEncoderType); - PyModule_AddObject(m, "make_encoder", (PyObject*)&PyEncoderType); - RawJSONType = import_dependency("simplejson.raw_json", "RawJSON"); - if (RawJSONType == NULL) - return NULL; - JSONDecodeError = import_dependency("simplejson.errors", "JSONDecodeError"); - if (JSONDecodeError == NULL) - return NULL; - return m; -} - -#if PY_MAJOR_VERSION >= 3 -PyMODINIT_FUNC -PyInit__speedups(void) -{ - return moduleinit(); -} -#else -void -init_speedups(void) -{ - moduleinit(); -} -#endif diff --git a/contrib/python/simplejson/py2/simplejson/compat.py b/contrib/python/simplejson/py2/simplejson/compat.py deleted file mode 100644 index 5fc14128443..00000000000 --- a/contrib/python/simplejson/py2/simplejson/compat.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Python 3 compatibility shims -""" -import sys -if sys.version_info[0] < 3: - PY3 = False - def b(s): - return s - try: - from cStringIO import StringIO - except ImportError: - from StringIO import StringIO - BytesIO = StringIO - text_type = unicode - binary_type = str - string_types = (basestring,) - integer_types = (int, long) - unichr = unichr - reload_module = reload -else: - PY3 = True - if sys.version_info[:2] >= (3, 4): - from importlib import reload as reload_module - else: - from imp import reload as reload_module - def b(s): - return bytes(s, 'latin1') - from io import StringIO, BytesIO - text_type = str - binary_type = bytes - string_types = (str,) - integer_types = (int,) - unichr = chr - -long_type = integer_types[-1] diff --git a/contrib/python/simplejson/py2/simplejson/decoder.py b/contrib/python/simplejson/py2/simplejson/decoder.py deleted file mode 100644 index c99a976d84e..00000000000 --- a/contrib/python/simplejson/py2/simplejson/decoder.py +++ /dev/null @@ -1,416 +0,0 @@ -"""Implementation of JSONDecoder -""" -from __future__ import absolute_import -import re -import sys -import struct -from .compat import PY3, unichr -from .scanner import make_scanner, JSONDecodeError - -def _import_c_scanstring(): - try: - from ._speedups import scanstring - return scanstring - except ImportError: - return None -c_scanstring = _import_c_scanstring() - -# NOTE (3.1.0): JSONDecodeError may still be imported from this module for -# compatibility, but it was never in the __all__ -__all__ = ['JSONDecoder'] - -FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL - -def _floatconstants(): - if sys.version_info < (2, 6): - _BYTES = '7FF80000000000007FF0000000000000'.decode('hex') - nan, inf = struct.unpack('>dd', _BYTES) - else: - nan = float('nan') - inf = float('inf') - return nan, inf, -inf - -NaN, PosInf, NegInf = _floatconstants() - -_CONSTANTS = { - '-Infinity': NegInf, - 'Infinity': PosInf, - 'NaN': NaN, -} - -STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS) -BACKSLASH = { - '"': u'"', '\\': u'\\', '/': u'/', - 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t', -} - -DEFAULT_ENCODING = "utf-8" - -if hasattr(sys, 'get_int_max_str_digits'): - bounded_int = int -else: - def bounded_int(s, INT_MAX_STR_DIGITS=4300): - """Backport of the integer string length conversion limitation - - https://docs.python.org/3/library/stdtypes.html#int-max-str-digits - """ - if len(s) > INT_MAX_STR_DIGITS: - raise ValueError("Exceeds the limit (%s) for integer string conversion: value has %s digits" % (INT_MAX_STR_DIGITS, len(s))) - return int(s) - - -def scan_four_digit_hex(s, end, _m=re.compile(r'^[0-9a-fA-F]{4}$').match): - """Scan a four digit hex number from s[end:end + 4] - """ - msg = "Invalid \\uXXXX escape sequence" - esc = s[end:end + 4] - if not _m(esc): - raise JSONDecodeError(msg, s, end - 2) - try: - return int(esc, 16), end + 4 - except ValueError: - raise JSONDecodeError(msg, s, end - 2) - -def py_scanstring(s, end, encoding=None, strict=True, - _b=BACKSLASH, _m=STRINGCHUNK.match, _join=u''.join, - _PY3=PY3, _maxunicode=sys.maxunicode, - _scan_four_digit_hex=scan_four_digit_hex): - """Scan the string s for a JSON string. End is the index of the - character in s after the quote that started the JSON string. - Unescapes all valid JSON string escape sequences and raises ValueError - on attempt to decode an invalid string. If strict is False then literal - control characters are allowed in the string. - - Returns a tuple of the decoded string and the index of the character in s - after the end quote.""" - if encoding is None: - encoding = DEFAULT_ENCODING - chunks = [] - _append = chunks.append - begin = end - 1 - while 1: - chunk = _m(s, end) - if chunk is None: - raise JSONDecodeError( - "Unterminated string starting at", s, begin) - prev_end = end - end = chunk.end() - content, terminator = chunk.groups() - # Content is contains zero or more unescaped string characters - if content: - if not _PY3 and not isinstance(content, unicode): - content = unicode(content, encoding) - _append(content) - # Terminator is the end of string, a literal control character, - # or a backslash denoting that an escape sequence follows - if terminator == '"': - break - elif terminator != '\\': - if strict: - msg = "Invalid control character %r at" - raise JSONDecodeError(msg, s, prev_end) - else: - _append(terminator) - continue - try: - esc = s[end] - except IndexError: - raise JSONDecodeError( - "Unterminated string starting at", s, begin) - # If not a unicode escape sequence, must be in the lookup table - if esc != 'u': - try: - char = _b[esc] - except KeyError: - msg = "Invalid \\X escape sequence %r" - raise JSONDecodeError(msg, s, end) - end += 1 - else: - # Unicode escape sequence - uni, end = _scan_four_digit_hex(s, end + 1) - # Check for surrogate pair on UCS-4 systems - # Note that this will join high/low surrogate pairs - # but will also pass unpaired surrogates through - if (_maxunicode > 65535 and - uni & 0xfc00 == 0xd800 and - s[end:end + 2] == '\\u'): - uni2, end2 = _scan_four_digit_hex(s, end + 2) - if uni2 & 0xfc00 == 0xdc00: - uni = 0x10000 + (((uni - 0xd800) << 10) | - (uni2 - 0xdc00)) - end = end2 - char = unichr(uni) - # Append the unescaped character - _append(char) - return _join(chunks), end - - -# Use speedup if available -scanstring = c_scanstring or py_scanstring - -WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS) -WHITESPACE_STR = ' \t\n\r' - -def JSONObject(state, encoding, strict, scan_once, object_hook, - object_pairs_hook, memo=None, - _w=WHITESPACE.match, _ws=WHITESPACE_STR): - (s, end) = state - # Backwards compatibility - if memo is None: - memo = {} - memo_get = memo.setdefault - pairs = [] - # Use a slice to prevent IndexError from being raised, the following - # check will raise a more specific ValueError if the string is empty - nextchar = s[end:end + 1] - # Normally we expect nextchar == '"' - if nextchar != '"': - if nextchar in _ws: - end = _w(s, end).end() - nextchar = s[end:end + 1] - # Trivial empty object - if nextchar == '}': - if object_pairs_hook is not None: - result = object_pairs_hook(pairs) - return result, end + 1 - pairs = {} - if object_hook is not None: - pairs = object_hook(pairs) - return pairs, end + 1 - elif nextchar != '"': - raise JSONDecodeError( - "Expecting property name enclosed in double quotes or '}'", - s, end) - end += 1 - while True: - key, end = scanstring(s, end, encoding, strict) - key = memo_get(key, key) - - # To skip some function call overhead we optimize the fast paths where - # the JSON key separator is ": " or just ":". - if s[end:end + 1] != ':': - end = _w(s, end).end() - if s[end:end + 1] != ':': - raise JSONDecodeError("Expecting ':' delimiter", s, end) - - end += 1 - - try: - if s[end] in _ws: - end += 1 - if s[end] in _ws: - end = _w(s, end + 1).end() - except IndexError: - pass - - value, end = scan_once(s, end) - pairs.append((key, value)) - - try: - nextchar = s[end] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end] - except IndexError: - nextchar = '' - end += 1 - - if nextchar == '}': - break - elif nextchar != ',': - raise JSONDecodeError("Expecting ',' delimiter or '}'", s, end - 1) - - try: - nextchar = s[end] - if nextchar in _ws: - end += 1 - nextchar = s[end] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end] - except IndexError: - nextchar = '' - - end += 1 - if nextchar != '"': - raise JSONDecodeError( - "Expecting property name enclosed in double quotes", - s, end - 1) - - if object_pairs_hook is not None: - result = object_pairs_hook(pairs) - return result, end - pairs = dict(pairs) - if object_hook is not None: - pairs = object_hook(pairs) - return pairs, end - -def JSONArray(state, scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): - (s, end) = state - values = [] - nextchar = s[end:end + 1] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end:end + 1] - # Look-ahead for trivial empty array - if nextchar == ']': - return values, end + 1 - elif nextchar == '': - raise JSONDecodeError("Expecting value or ']'", s, end) - _append = values.append - while True: - value, end = scan_once(s, end) - _append(value) - nextchar = s[end:end + 1] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end:end + 1] - end += 1 - if nextchar == ']': - break - elif nextchar != ',': - raise JSONDecodeError("Expecting ',' delimiter or ']'", s, end - 1) - - try: - if s[end] in _ws: - end += 1 - if s[end] in _ws: - end = _w(s, end + 1).end() - except IndexError: - pass - - return values, end - -class JSONDecoder(object): - """Simple JSON <http://json.org> decoder - - Performs the following translations in decoding by default: - - +---------------+-------------------+ - | JSON | Python | - +===============+===================+ - | object | dict | - +---------------+-------------------+ - | array | list | - +---------------+-------------------+ - | string | str, unicode | - +---------------+-------------------+ - | number (int) | int, long | - +---------------+-------------------+ - | number (real) | float | - +---------------+-------------------+ - | true | True | - +---------------+-------------------+ - | false | False | - +---------------+-------------------+ - | null | None | - +---------------+-------------------+ - - When allow_nan=True, it also understands - ``NaN``, ``Infinity``, and ``-Infinity`` as - their corresponding ``float`` values, which is outside the JSON spec. - - """ - - def __init__(self, encoding=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, strict=True, - object_pairs_hook=None, allow_nan=False): - """ - *encoding* determines the encoding used to interpret any - :class:`str` objects decoded by this instance (``'utf-8'`` by - default). It has no effect when decoding :class:`unicode` objects. - - Note that currently only encodings that are a superset of ASCII work, - strings of other encodings should be passed in as :class:`unicode`. - - *object_hook*, if specified, will be called with the result of every - JSON object decoded and its return value will be used in place of the - given :class:`dict`. This can be used to provide custom - deserializations (e.g. to support JSON-RPC class hinting). - - *object_pairs_hook* is an optional function that will be called with - the result of any object literal decode with an ordered list of pairs. - The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders - that rely on the order that the key and value pairs are decoded (for - example, :func:`collections.OrderedDict` will remember the order of - insertion). If *object_hook* is also defined, the *object_pairs_hook* - takes priority. - - *parse_float*, if specified, will be called with the string of every - JSON float to be decoded. By default, this is equivalent to - ``float(num_str)``. This can be used to use another datatype or parser - for JSON floats (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every - JSON int to be decoded. By default, this is equivalent to - ``int(num_str)``. This can be used to use another datatype or parser - for JSON integers (e.g. :class:`float`). - - *allow_nan*, if True (default false), will allow the parser to - accept the non-standard floats ``NaN``, ``Infinity``, and ``-Infinity``. - - *parse_constant*, if specified, will be - called with one of the following strings: ``'-Infinity'``, - ``'Infinity'``, ``'NaN'``. It is not recommended to use this feature, - as it is rare to parse non-compliant JSON containing these values. - - *strict* controls the parser's behavior when it encounters an - invalid control character in a string. The default setting of - ``True`` means that unescaped control characters are parse errors, if - ``False`` then control characters will be allowed in strings. - - """ - if encoding is None: - encoding = DEFAULT_ENCODING - self.encoding = encoding - self.object_hook = object_hook - self.object_pairs_hook = object_pairs_hook - self.parse_float = parse_float or float - self.parse_int = parse_int or bounded_int - self.parse_constant = parse_constant or (allow_nan and _CONSTANTS.__getitem__ or None) - self.strict = strict - self.parse_object = JSONObject - self.parse_array = JSONArray - self.parse_string = scanstring - self.memo = {} - self.scan_once = make_scanner(self) - - def decode(self, s, _w=WHITESPACE.match, _PY3=PY3): - """Return the Python representation of ``s`` (a ``str`` or ``unicode`` - instance containing a JSON document) - - """ - if _PY3 and isinstance(s, bytes): - s = str(s, self.encoding) - obj, end = self.raw_decode(s) - end = _w(s, end).end() - if end != len(s): - raise JSONDecodeError("Extra data", s, end, len(s)) - return obj - - def raw_decode(self, s, idx=0, _w=WHITESPACE.match, _PY3=PY3): - """Decode a JSON document from ``s`` (a ``str`` or ``unicode`` - beginning with a JSON document) and return a 2-tuple of the Python - representation and the index in ``s`` where the document ended. - Optionally, ``idx`` can be used to specify an offset in ``s`` where - the JSON document begins. - - This can be used to decode a JSON document from a string that may - have extraneous data at the end. - - """ - if idx < 0: - # Ensure that raw_decode bails on negative indexes, the regex - # would otherwise mask this behavior. #98 - raise JSONDecodeError('Expecting value', s, idx) - if _PY3 and not isinstance(s, str): - raise TypeError("Input string must be text, not bytes") - # strip UTF-8 bom - if len(s) > idx: - ord0 = ord(s[idx]) - if ord0 == 0xfeff: - idx += 1 - elif ord0 == 0xef and s[idx:idx + 3] == '\xef\xbb\xbf': - idx += 3 - return self.scan_once(s, idx=_w(s, idx).end()) diff --git a/contrib/python/simplejson/py2/simplejson/encoder.py b/contrib/python/simplejson/py2/simplejson/encoder.py deleted file mode 100644 index 661ff361bf2..00000000000 --- a/contrib/python/simplejson/py2/simplejson/encoder.py +++ /dev/null @@ -1,740 +0,0 @@ -"""Implementation of JSONEncoder -""" -from __future__ import absolute_import -import re -from operator import itemgetter -# Do not import Decimal directly to avoid reload issues -import decimal -from .compat import binary_type, text_type, string_types, integer_types, PY3 -def _import_speedups(): - try: - from . import _speedups - return _speedups.encode_basestring_ascii, _speedups.make_encoder - except ImportError: - return None, None -c_encode_basestring_ascii, c_make_encoder = _import_speedups() - -from .decoder import PosInf -from .raw_json import RawJSON - -ESCAPE = re.compile(r'[\x00-\x1f\\"]') -ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') -HAS_UTF8 = re.compile(r'[\x80-\xff]') -ESCAPE_DCT = { - '\\': '\\\\', - '"': '\\"', - '\b': '\\b', - '\f': '\\f', - '\n': '\\n', - '\r': '\\r', - '\t': '\\t', -} -for i in range(0x20): - #ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i)) - ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,)) -del i - -FLOAT_REPR = repr - -def encode_basestring(s, _PY3=PY3, _q=u'"'): - """Return a JSON representation of a Python string - - """ - if _PY3: - if isinstance(s, bytes): - s = str(s, 'utf-8') - elif type(s) is not str: - # convert an str subclass instance to exact str - # raise a TypeError otherwise - s = str.__str__(s) - else: - if isinstance(s, str) and HAS_UTF8.search(s) is not None: - s = unicode(s, 'utf-8') - elif type(s) not in (str, unicode): - # convert an str subclass instance to exact str - # convert a unicode subclass instance to exact unicode - # raise a TypeError otherwise - if isinstance(s, str): - s = str.__str__(s) - else: - s = unicode.__getnewargs__(s)[0] - def replace(match): - return ESCAPE_DCT[match.group(0)] - return _q + ESCAPE.sub(replace, s) + _q - - -def py_encode_basestring_ascii(s, _PY3=PY3): - """Return an ASCII-only JSON representation of a Python string - - """ - if _PY3: - if isinstance(s, bytes): - s = str(s, 'utf-8') - elif type(s) is not str: - # convert an str subclass instance to exact str - # raise a TypeError otherwise - s = str.__str__(s) - else: - if isinstance(s, str) and HAS_UTF8.search(s) is not None: - s = unicode(s, 'utf-8') - elif type(s) not in (str, unicode): - # convert an str subclass instance to exact str - # convert a unicode subclass instance to exact unicode - # raise a TypeError otherwise - if isinstance(s, str): - s = str.__str__(s) - else: - s = unicode.__getnewargs__(s)[0] - def replace(match): - s = match.group(0) - try: - return ESCAPE_DCT[s] - except KeyError: - n = ord(s) - if n < 0x10000: - #return '\\u{0:04x}'.format(n) - return '\\u%04x' % (n,) - else: - # surrogate pair - n -= 0x10000 - s1 = 0xd800 | ((n >> 10) & 0x3ff) - s2 = 0xdc00 | (n & 0x3ff) - #return '\\u{0:04x}\\u{1:04x}'.format(s1, s2) - return '\\u%04x\\u%04x' % (s1, s2) - return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"' - - -encode_basestring_ascii = ( - c_encode_basestring_ascii or py_encode_basestring_ascii) - -class JSONEncoder(object): - """Extensible JSON <http://json.org> encoder for Python data structures. - - Supports the following objects and types by default: - - +-------------------+---------------+ - | Python | JSON | - +===================+===============+ - | dict, namedtuple | object | - +-------------------+---------------+ - | list, tuple | array | - +-------------------+---------------+ - | str, unicode | string | - +-------------------+---------------+ - | int, long, float | number | - +-------------------+---------------+ - | True | true | - +-------------------+---------------+ - | False | false | - +-------------------+---------------+ - | None | null | - +-------------------+---------------+ - - To extend this to recognize other objects, subclass and implement a - ``.default()`` method with another method that returns a serializable - object for ``o`` if possible, otherwise it should call the superclass - implementation (to raise ``TypeError``). - - """ - item_separator = ', ' - key_separator = ': ' - - def __init__(self, skipkeys=False, ensure_ascii=True, - check_circular=True, allow_nan=False, sort_keys=False, - indent=None, separators=None, encoding='utf-8', default=None, - use_decimal=True, namedtuple_as_object=True, - tuple_as_array=True, bigint_as_string=False, - item_sort_key=None, for_json=False, ignore_nan=False, - int_as_string_bitcount=None, iterable_as_array=False): - """Constructor for JSONEncoder, with sensible defaults. - - If skipkeys is false, then it is a TypeError to attempt - encoding of keys that are not str, int, long, float or None. If - skipkeys is True, such items are simply skipped. - - If ensure_ascii is true, the output is guaranteed to be str - objects with all incoming unicode characters escaped. If - ensure_ascii is false, the output will be unicode object. - - If check_circular is true, then lists, dicts, and custom encoded - objects will be checked for circular references during encoding to - prevent an infinite recursion (which would cause an OverflowError). - Otherwise, no such check takes place. - - If allow_nan is true (default: False), then out of range float - values (nan, inf, -inf) will be serialized to - their JavaScript equivalents (NaN, Infinity, -Infinity) - instead of raising a ValueError. See - ignore_nan for ECMA-262 compliant behavior. - - If sort_keys is true, then the output of dictionaries will be - sorted by key; this is useful for regression tests to ensure - that JSON serializations can be compared on a day-to-day basis. - - If indent is a string, then JSON array elements and object members - will be pretty-printed with a newline followed by that string repeated - for each level of nesting. ``None`` (the default) selects the most compact - representation without any newlines. For backwards compatibility with - versions of simplejson earlier than 2.1.0, an integer is also accepted - and is converted to a string with that many spaces. - - If specified, separators should be an (item_separator, key_separator) - tuple. The default is (', ', ': ') if *indent* is ``None`` and - (',', ': ') otherwise. To get the most compact JSON representation, - you should specify (',', ':') to eliminate whitespace. - - If specified, default is a function that gets called for objects - that can't otherwise be serialized. It should return a JSON encodable - version of the object or raise a ``TypeError``. - - If encoding is not None, then all input strings will be - transformed into unicode using that encoding prior to JSON-encoding. - The default is UTF-8. - - If use_decimal is true (default: ``True``), ``decimal.Decimal`` will - be supported directly by the encoder. For the inverse, decode JSON - with ``parse_float=decimal.Decimal``. - - If namedtuple_as_object is true (the default), objects with - ``_asdict()`` methods will be encoded as JSON objects. - - If tuple_as_array is true (the default), tuple (and subclasses) will - be encoded as JSON arrays. - - If *iterable_as_array* is true (default: ``False``), - any object not in the above table that implements ``__iter__()`` - will be encoded as a JSON array. - - If bigint_as_string is true (not the default), ints 2**53 and higher - or lower than -2**53 will be encoded as strings. This is to avoid the - rounding that happens in Javascript otherwise. - - If int_as_string_bitcount is a positive number (n), then int of size - greater than or equal to 2**n or lower than or equal to -2**n will be - encoded as strings. - - If specified, item_sort_key is a callable used to sort the items in - each dictionary. This is useful if you want to sort items other than - in alphabetical order by key. - - If for_json is true (not the default), objects with a ``for_json()`` - method will use the return value of that method for encoding as JSON - instead of the object. - - If *ignore_nan* is true (default: ``False``), then out of range - :class:`float` values (``nan``, ``inf``, ``-inf``) will be serialized - as ``null`` in compliance with the ECMA-262 specification. If true, - this will override *allow_nan*. - - """ - - self.skipkeys = skipkeys - self.ensure_ascii = ensure_ascii - self.check_circular = check_circular - self.allow_nan = allow_nan - self.sort_keys = sort_keys - self.use_decimal = use_decimal - self.namedtuple_as_object = namedtuple_as_object - self.tuple_as_array = tuple_as_array - self.iterable_as_array = iterable_as_array - self.bigint_as_string = bigint_as_string - self.item_sort_key = item_sort_key - self.for_json = for_json - self.ignore_nan = ignore_nan - self.int_as_string_bitcount = int_as_string_bitcount - if indent is not None and not isinstance(indent, string_types): - indent = indent * ' ' - self.indent = indent - if separators is not None: - self.item_separator, self.key_separator = separators - elif indent is not None: - self.item_separator = ',' - if default is not None: - self.default = default - self.encoding = encoding - - def default(self, o): - """Implement this method in a subclass such that it returns - a serializable object for ``o``, or calls the base implementation - (to raise a ``TypeError``). - - For example, to support arbitrary iterators, you could - implement default like this:: - - def default(self, o): - try: - iterable = iter(o) - except TypeError: - pass - else: - return list(iterable) - return JSONEncoder.default(self, o) - - """ - raise TypeError('Object of type %s is not JSON serializable' % - o.__class__.__name__) - - def encode(self, o): - """Return a JSON string representation of a Python data structure. - - >>> from simplejson import JSONEncoder - >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) - '{"foo": ["bar", "baz"]}' - - """ - # This is for extremely simple cases and benchmarks. - if isinstance(o, binary_type): - _encoding = self.encoding - if (_encoding is not None and not (_encoding == 'utf-8')): - o = text_type(o, _encoding) - if isinstance(o, string_types): - if self.ensure_ascii: - return encode_basestring_ascii(o) - else: - return encode_basestring(o) - # This doesn't pass the iterator directly to ''.join() because the - # exceptions aren't as detailed. The list call should be roughly - # equivalent to the PySequence_Fast that ''.join() would do. - chunks = self.iterencode(o) - if not isinstance(chunks, (list, tuple)): - chunks = list(chunks) - if self.ensure_ascii: - return ''.join(chunks) - else: - return u''.join(chunks) - - def iterencode(self, o): - """Encode the given object and yield each string - representation as available. - - For example:: - - for chunk in JSONEncoder().iterencode(bigobject): - mysocket.write(chunk) - - """ - if self.check_circular: - markers = {} - else: - markers = None - if self.ensure_ascii: - _encoder = encode_basestring_ascii - else: - _encoder = encode_basestring - if self.encoding != 'utf-8' and self.encoding is not None: - def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding): - if isinstance(o, binary_type): - o = text_type(o, _encoding) - return _orig_encoder(o) - - def floatstr(o, allow_nan=self.allow_nan, ignore_nan=self.ignore_nan, - _repr=FLOAT_REPR, _inf=PosInf, _neginf=-PosInf): - # Check for specials. Note that this type of test is processor - # and/or platform-specific, so do tests which don't depend on - # the internals. - - if o != o: - text = 'NaN' - elif o == _inf: - text = 'Infinity' - elif o == _neginf: - text = '-Infinity' - else: - if type(o) != float: - # See #118, do not trust custom str/repr - o = float(o) - return _repr(o) - - if ignore_nan: - text = 'null' - elif not allow_nan: - raise ValueError( - "Out of range float values are not JSON compliant: " + - repr(o)) - - return text - - key_memo = {} - int_as_string_bitcount = ( - 53 if self.bigint_as_string else self.int_as_string_bitcount) - if (c_make_encoder is not None and self.indent is None): - _iterencode = c_make_encoder( - markers, self.default, _encoder, self.indent, - self.key_separator, self.item_separator, self.sort_keys, - self.skipkeys, self.allow_nan, key_memo, self.use_decimal, - self.namedtuple_as_object, self.tuple_as_array, - int_as_string_bitcount, - self.item_sort_key, self.encoding, self.for_json, - self.ignore_nan, decimal.Decimal, self.iterable_as_array) - else: - _iterencode = _make_iterencode( - markers, self.default, _encoder, self.indent, floatstr, - self.key_separator, self.item_separator, self.sort_keys, - self.skipkeys, self.use_decimal, - self.namedtuple_as_object, self.tuple_as_array, - int_as_string_bitcount, - self.item_sort_key, self.encoding, self.for_json, - self.iterable_as_array, Decimal=decimal.Decimal) - try: - return _iterencode(o, 0) - finally: - key_memo.clear() - - -class JSONEncoderForHTML(JSONEncoder): - """An encoder that produces JSON safe to embed in HTML. - - To embed JSON content in, say, a script tag on a web page, the - characters &, < and > should be escaped. They cannot be escaped - with the usual entities (e.g. &) because they are not expanded - within <script> tags. - - This class also escapes the line separator and paragraph separator - characters U+2028 and U+2029, irrespective of the ensure_ascii setting, - as these characters are not valid in JavaScript strings (see - http://timelessrepo.com/json-isnt-a-javascript-subset). - """ - - def encode(self, o): - # Override JSONEncoder.encode because it has hacks for - # performance that make things more complicated. - chunks = self.iterencode(o) - if self.ensure_ascii: - return ''.join(chunks) - else: - return u''.join(chunks) - - def iterencode(self, o): - chunks = super(JSONEncoderForHTML, self).iterencode(o) - for chunk in chunks: - chunk = chunk.replace('&', '\\u0026') - chunk = chunk.replace('<', '\\u003c') - chunk = chunk.replace('>', '\\u003e') - - if not self.ensure_ascii: - chunk = chunk.replace(u'\u2028', '\\u2028') - chunk = chunk.replace(u'\u2029', '\\u2029') - - yield chunk - - -def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, - _key_separator, _item_separator, _sort_keys, _skipkeys, - _use_decimal, _namedtuple_as_object, _tuple_as_array, - _int_as_string_bitcount, _item_sort_key, - _encoding,_for_json, - _iterable_as_array, - ## HACK: hand-optimized bytecode; turn globals into locals - _PY3=PY3, - ValueError=ValueError, - string_types=string_types, - Decimal=None, - dict=dict, - float=float, - id=id, - integer_types=integer_types, - isinstance=isinstance, - list=list, - str=str, - tuple=tuple, - iter=iter, - ): - if _use_decimal and Decimal is None: - Decimal = decimal.Decimal - if _item_sort_key and not callable(_item_sort_key): - raise TypeError("item_sort_key must be None or callable") - elif _sort_keys and not _item_sort_key: - _item_sort_key = itemgetter(0) - - if (_int_as_string_bitcount is not None and - (_int_as_string_bitcount <= 0 or - not isinstance(_int_as_string_bitcount, integer_types))): - raise TypeError("int_as_string_bitcount must be a positive integer") - - def call_method(obj, method_name): - method = getattr(obj, method_name, None) - if callable(method): - try: - return (method(),) - except TypeError: - pass - return None - - def _encode_int(value): - skip_quoting = ( - _int_as_string_bitcount is None - or - _int_as_string_bitcount < 1 - ) - if type(value) not in integer_types: - # See #118, do not trust custom str/repr - value = int(value) - if ( - skip_quoting or - (-1 << _int_as_string_bitcount) - < value < - (1 << _int_as_string_bitcount) - ): - return str(value) - return '"' + str(value) + '"' - - def _iterencode_list(lst, _current_indent_level): - if not lst: - yield '[]' - return - if markers is not None: - markerid = id(lst) - if markerid in markers: - raise ValueError("Circular reference detected") - markers[markerid] = lst - buf = '[' - if _indent is not None: - _current_indent_level += 1 - newline_indent = '\n' + (_indent * _current_indent_level) - separator = _item_separator + newline_indent - buf += newline_indent - else: - newline_indent = None - separator = _item_separator - first = True - for value in lst: - if first: - first = False - else: - buf = separator - if isinstance(value, string_types): - yield buf + _encoder(value) - elif _PY3 and isinstance(value, bytes) and _encoding is not None: - yield buf + _encoder(value) - elif isinstance(value, RawJSON): - yield buf + value.encoded_json - elif value is None: - yield buf + 'null' - elif value is True: - yield buf + 'true' - elif value is False: - yield buf + 'false' - elif isinstance(value, integer_types): - yield buf + _encode_int(value) - elif isinstance(value, float): - yield buf + _floatstr(value) - elif _use_decimal and isinstance(value, Decimal): - yield buf + str(value) - else: - yield buf - for_json = _for_json and call_method(value, 'for_json') - if for_json: - chunks = _iterencode(for_json[0], _current_indent_level) - elif isinstance(value, list): - chunks = _iterencode_list(value, _current_indent_level) - else: - _asdict = _namedtuple_as_object and call_method(value, '_asdict') - if _asdict: - dct = _asdict[0] - if not isinstance(dct, dict): - raise TypeError("_asdict() must return a dict, not %s" % (type(dct).__name__,)) - chunks = _iterencode_dict(dct, - _current_indent_level) - elif _tuple_as_array and isinstance(value, tuple): - chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): - chunks = _iterencode_dict(value, _current_indent_level) - else: - chunks = _iterencode(value, _current_indent_level) - for chunk in chunks: - yield chunk - if first: - # iterable_as_array misses the fast path at the top - yield '[]' - else: - if newline_indent is not None: - _current_indent_level -= 1 - yield '\n' + (_indent * _current_indent_level) - yield ']' - if markers is not None: - del markers[markerid] - - def _stringify_key(key): - if isinstance(key, string_types): # pragma: no cover - pass - elif _PY3 and isinstance(key, bytes) and _encoding is not None: - key = str(key, _encoding) - elif isinstance(key, float): - key = _floatstr(key) - elif key is True: - key = 'true' - elif key is False: - key = 'false' - elif key is None: - key = 'null' - elif isinstance(key, integer_types): - if type(key) not in integer_types: - # See #118, do not trust custom str/repr - key = int(key) - key = str(key) - elif _use_decimal and isinstance(key, Decimal): - key = str(key) - elif _skipkeys: - key = None - else: - raise TypeError('keys must be str, int, float, bool or None, ' - 'not %s' % key.__class__.__name__) - return key - - def _iterencode_dict(dct, _current_indent_level): - if not dct: - yield '{}' - return - if markers is not None: - markerid = id(dct) - if markerid in markers: - raise ValueError("Circular reference detected") - markers[markerid] = dct - yield '{' - if _indent is not None: - _current_indent_level += 1 - newline_indent = '\n' + (_indent * _current_indent_level) - item_separator = _item_separator + newline_indent - yield newline_indent - else: - newline_indent = None - item_separator = _item_separator - first = True - if _PY3: - iteritems = dct.items() - else: - iteritems = dct.iteritems() - if _item_sort_key: - items = [] - for k, v in dct.items(): - if not isinstance(k, string_types): - k = _stringify_key(k) - if k is None: - continue - items.append((k, v)) - items.sort(key=_item_sort_key) - else: - items = iteritems - for key, value in items: - if not (_item_sort_key or isinstance(key, string_types)): - key = _stringify_key(key) - if key is None: - # _skipkeys must be True - continue - if first: - first = False - else: - yield item_separator - yield _encoder(key) - yield _key_separator - if isinstance(value, string_types): - yield _encoder(value) - elif _PY3 and isinstance(value, bytes) and _encoding is not None: - yield _encoder(value) - elif isinstance(value, RawJSON): - yield value.encoded_json - elif value is None: - yield 'null' - elif value is True: - yield 'true' - elif value is False: - yield 'false' - elif isinstance(value, integer_types): - yield _encode_int(value) - elif isinstance(value, float): - yield _floatstr(value) - elif _use_decimal and isinstance(value, Decimal): - yield str(value) - else: - for_json = _for_json and call_method(value, 'for_json') - if for_json: - chunks = _iterencode(for_json[0], _current_indent_level) - elif isinstance(value, list): - chunks = _iterencode_list(value, _current_indent_level) - else: - _asdict = _namedtuple_as_object and call_method(value, '_asdict') - if _asdict: - dct = _asdict[0] - if not isinstance(dct, dict): - raise TypeError("_asdict() must return a dict, not %s" % (type(dct).__name__,)) - chunks = _iterencode_dict(dct, - _current_indent_level) - elif _tuple_as_array and isinstance(value, tuple): - chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): - chunks = _iterencode_dict(value, _current_indent_level) - else: - chunks = _iterencode(value, _current_indent_level) - for chunk in chunks: - yield chunk - if newline_indent is not None: - _current_indent_level -= 1 - yield '\n' + (_indent * _current_indent_level) - yield '}' - if markers is not None: - del markers[markerid] - - def _iterencode(o, _current_indent_level): - if isinstance(o, string_types): - yield _encoder(o) - elif _PY3 and isinstance(o, bytes) and _encoding is not None: - yield _encoder(o) - elif isinstance(o, RawJSON): - yield o.encoded_json - elif o is None: - yield 'null' - elif o is True: - yield 'true' - elif o is False: - yield 'false' - elif isinstance(o, integer_types): - yield _encode_int(o) - elif isinstance(o, float): - yield _floatstr(o) - else: - for_json = _for_json and call_method(o, 'for_json') - if for_json: - for chunk in _iterencode(for_json[0], _current_indent_level): - yield chunk - elif isinstance(o, list): - for chunk in _iterencode_list(o, _current_indent_level): - yield chunk - else: - _asdict = _namedtuple_as_object and call_method(o, '_asdict') - if _asdict: - dct = _asdict[0] - if not isinstance(dct, dict): - raise TypeError("_asdict() must return a dict, not %s" % (type(dct).__name__,)) - for chunk in _iterencode_dict(dct, _current_indent_level): - yield chunk - elif (_tuple_as_array and isinstance(o, tuple)): - for chunk in _iterencode_list(o, _current_indent_level): - yield chunk - elif isinstance(o, dict): - for chunk in _iterencode_dict(o, _current_indent_level): - yield chunk - elif _use_decimal and isinstance(o, Decimal): - yield str(o) - else: - while _iterable_as_array: - # Markers are not checked here because it is valid for - # an iterable to return self. - try: - o = iter(o) - except TypeError: - break - for chunk in _iterencode_list(o, _current_indent_level): - yield chunk - return - if markers is not None: - markerid = id(o) - if markerid in markers: - raise ValueError("Circular reference detected") - markers[markerid] = o - o = _default(o) - for chunk in _iterencode(o, _current_indent_level): - yield chunk - if markers is not None: - del markers[markerid] - - return _iterencode diff --git a/contrib/python/simplejson/py2/simplejson/errors.py b/contrib/python/simplejson/py2/simplejson/errors.py deleted file mode 100644 index b97ab1e913f..00000000000 --- a/contrib/python/simplejson/py2/simplejson/errors.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Error classes used by simplejson -""" -__all__ = ['JSONDecodeError'] - - -def linecol(doc, pos): - lineno = doc.count('\n', 0, pos) + 1 - if lineno == 1: - colno = pos + 1 - else: - colno = pos - doc.rindex('\n', 0, pos) - return lineno, colno - - -def errmsg(msg, doc, pos, end=None): - lineno, colno = linecol(doc, pos) - msg = msg.replace('%r', repr(doc[pos:pos + 1])) - if end is None: - fmt = '%s: line %d column %d (char %d)' - return fmt % (msg, lineno, colno, pos) - endlineno, endcolno = linecol(doc, end) - fmt = '%s: line %d column %d - line %d column %d (char %d - %d)' - return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end) - - -class JSONDecodeError(ValueError): - """Subclass of ValueError with the following additional properties: - - msg: The unformatted error message - doc: The JSON document being parsed - pos: The start index of doc where parsing failed - end: The end index of doc where parsing failed (may be None) - lineno: The line corresponding to pos - colno: The column corresponding to pos - endlineno: The line corresponding to end (may be None) - endcolno: The column corresponding to end (may be None) - - """ - # Note that this exception is used from _speedups - def __init__(self, msg, doc, pos, end=None): - ValueError.__init__(self, errmsg(msg, doc, pos, end=end)) - self.msg = msg - self.doc = doc - self.pos = pos - self.end = end - self.lineno, self.colno = linecol(doc, pos) - if end is not None: - self.endlineno, self.endcolno = linecol(doc, end) - else: - self.endlineno, self.endcolno = None, None - - def __reduce__(self): - return self.__class__, (self.msg, self.doc, self.pos, self.end) diff --git a/contrib/python/simplejson/py2/simplejson/ordered_dict.py b/contrib/python/simplejson/py2/simplejson/ordered_dict.py deleted file mode 100644 index d5a55ebd0f7..00000000000 --- a/contrib/python/simplejson/py2/simplejson/ordered_dict.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Drop-in replacement for collections.OrderedDict by Raymond Hettinger - -http://code.activestate.com/recipes/576693/ - -""" -from UserDict import DictMixin - -class OrderedDict(dict, DictMixin): - - def __init__(self, *args, **kwds): - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__end - except AttributeError: - self.clear() - self.update(*args, **kwds) - - def clear(self): - self.__end = end = [] - end += [None, end, end] # sentinel node for doubly linked list - self.__map = {} # key --> [key, prev, next] - dict.clear(self) - - def __setitem__(self, key, value): - if key not in self: - end = self.__end - curr = end[1] - curr[2] = end[1] = self.__map[key] = [key, curr, end] - dict.__setitem__(self, key, value) - - def __delitem__(self, key): - dict.__delitem__(self, key) - key, prev, next = self.__map.pop(key) - prev[2] = next - next[1] = prev - - def __iter__(self): - end = self.__end - curr = end[2] - while curr is not end: - yield curr[0] - curr = curr[2] - - def __reversed__(self): - end = self.__end - curr = end[1] - while curr is not end: - yield curr[0] - curr = curr[1] - - def popitem(self, last=True): - if not self: - raise KeyError('dictionary is empty') - key = reversed(self).next() if last else iter(self).next() - value = self.pop(key) - return key, value - - def __reduce__(self): - items = [[k, self[k]] for k in self] - tmp = self.__map, self.__end - del self.__map, self.__end - inst_dict = vars(self).copy() - self.__map, self.__end = tmp - if inst_dict: - return (self.__class__, (items,), inst_dict) - return self.__class__, (items,) - - def keys(self): - return list(self) - - setdefault = DictMixin.setdefault - update = DictMixin.update - pop = DictMixin.pop - values = DictMixin.values - items = DictMixin.items - iterkeys = DictMixin.iterkeys - itervalues = DictMixin.itervalues - iteritems = DictMixin.iteritems - - def __repr__(self): - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, self.items()) - - def copy(self): - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - d = cls() - for key in iterable: - d[key] = value - return d - - def __eq__(self, other): - if isinstance(other, OrderedDict): - return len(self)==len(other) and \ - all(p==q for p, q in zip(self.items(), other.items())) - return dict.__eq__(self, other) - - def __ne__(self, other): - return not self == other diff --git a/contrib/python/simplejson/py2/simplejson/raw_json.py b/contrib/python/simplejson/py2/simplejson/raw_json.py deleted file mode 100644 index 2071a70206b..00000000000 --- a/contrib/python/simplejson/py2/simplejson/raw_json.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Implementation of RawJSON -""" - -class RawJSON(object): - """Wrap an encoded JSON document for direct embedding in the output - - """ - def __init__(self, encoded_json): - self.encoded_json = encoded_json diff --git a/contrib/python/simplejson/py2/simplejson/scanner.py b/contrib/python/simplejson/py2/simplejson/scanner.py deleted file mode 100644 index 34710d68c6d..00000000000 --- a/contrib/python/simplejson/py2/simplejson/scanner.py +++ /dev/null @@ -1,85 +0,0 @@ -"""JSON token scanner -""" -import re -from .errors import JSONDecodeError -def _import_c_make_scanner(): - try: - from ._speedups import make_scanner - return make_scanner - except ImportError: - return None -c_make_scanner = _import_c_make_scanner() - -__all__ = ['make_scanner', 'JSONDecodeError'] - -NUMBER_RE = re.compile( - r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?', - (re.VERBOSE | re.MULTILINE | re.DOTALL)) - - -def py_make_scanner(context): - parse_object = context.parse_object - parse_array = context.parse_array - parse_string = context.parse_string - match_number = NUMBER_RE.match - encoding = context.encoding - strict = context.strict - parse_float = context.parse_float - parse_int = context.parse_int - parse_constant = context.parse_constant - object_hook = context.object_hook - object_pairs_hook = context.object_pairs_hook - memo = context.memo - - def _scan_once(string, idx): - errmsg = 'Expecting value' - try: - nextchar = string[idx] - except IndexError: - raise JSONDecodeError(errmsg, string, idx) - - if nextchar == '"': - return parse_string(string, idx + 1, encoding, strict) - elif nextchar == '{': - return parse_object((string, idx + 1), encoding, strict, - _scan_once, object_hook, object_pairs_hook, memo) - elif nextchar == '[': - return parse_array((string, idx + 1), _scan_once) - elif nextchar == 'n' and string[idx:idx + 4] == 'null': - return None, idx + 4 - elif nextchar == 't' and string[idx:idx + 4] == 'true': - return True, idx + 4 - elif nextchar == 'f' and string[idx:idx + 5] == 'false': - return False, idx + 5 - - m = match_number(string, idx) - if m is not None: - integer, frac, exp = m.groups() - if frac or exp: - res = parse_float(integer + (frac or '') + (exp or '')) - else: - res = parse_int(integer) - return res, m.end() - elif parse_constant and nextchar == 'N' and string[idx:idx + 3] == 'NaN': - return parse_constant('NaN'), idx + 3 - elif parse_constant and nextchar == 'I' and string[idx:idx + 8] == 'Infinity': - return parse_constant('Infinity'), idx + 8 - elif parse_constant and nextchar == '-' and string[idx:idx + 9] == '-Infinity': - return parse_constant('-Infinity'), idx + 9 - else: - raise JSONDecodeError(errmsg, string, idx) - - def scan_once(string, idx): - if idx < 0: - # Ensure the same behavior as the C speedup, otherwise - # this would work for *some* negative string indices due - # to the behavior of __getitem__ for strings. #98 - raise JSONDecodeError('Expecting value', string, idx) - try: - return _scan_once(string, idx) - finally: - memo.clear() - - return scan_once - -make_scanner = c_make_scanner or py_make_scanner diff --git a/contrib/python/simplejson/py2/simplejson/tests/__init__.py b/contrib/python/simplejson/py2/simplejson/tests/__init__.py deleted file mode 100644 index d9986d033a0..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/__init__.py +++ /dev/null @@ -1,93 +0,0 @@ -from __future__ import absolute_import -import unittest -import sys -import os - - -class NoExtensionTestSuite(unittest.TestSuite): - def run(self, result): - import simplejson - - simplejson._toggle_speedups(False) - result = unittest.TestSuite.run(self, result) - simplejson._toggle_speedups(True) - return result - - -class TestMissingSpeedups(unittest.TestCase): - def runTest(self): - if hasattr(sys, "pypy_translation_info"): - "PyPy doesn't need speedups! :)" - elif getattr(getattr(sys, "implementation", None), "name", None) == "graalpy": - "GraalPy doesn't need speedups! :)" - elif hasattr(self, "skipTest"): - self.skipTest("_speedups.so is missing!") - - -def additional_tests(suite=None, project_dir=None): - import simplejson - import simplejson.encoder - import simplejson.decoder - - if suite is None: - suite = unittest.TestSuite() - try: - import doctest - except ImportError: - if sys.version_info < (2, 7): - # doctests in 2.6 depends on cStringIO - return suite - raise - for mod in (simplejson, simplejson.encoder, simplejson.decoder): - suite.addTest(doctest.DocTestSuite(mod)) - if project_dir is not None: - suite.addTest( - doctest.DocFileSuite( - os.path.join(project_dir, "index.rst"), module_relative=False - ) - ) - return suite - - -def all_tests_suite(project_dir=None): - def get_suite(): - suite_names = [ - "simplejson.tests.%s" % (os.path.splitext(f)[0],) - for f in os.listdir(os.path.dirname(__file__)) - if f.startswith("test_") and f.endswith(".py") - ] - return additional_tests( - suite=unittest.TestLoader().loadTestsFromNames(suite_names), - project_dir=project_dir, - ) - - suite = get_suite() - import simplejson - - if simplejson._import_c_make_encoder() is None: - suite.addTest(TestMissingSpeedups()) - else: - suite = unittest.TestSuite( - [ - suite, - NoExtensionTestSuite([get_suite()]), - ] - ) - return suite - - -def main(project_dir=None): - runner = unittest.TextTestRunner(verbosity=1 + sys.argv.count("-v")) - suite = all_tests_suite(project_dir=project_dir) - raise SystemExit(not runner.run(suite).wasSuccessful()) - - -if __name__ == "__main__": - import os - import sys - - project_dir = os.path.dirname( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - ) - sys.path.insert(0, project_dir) - main(project_dir=project_dir) diff --git a/contrib/python/simplejson/py2/simplejson/tests/_cibw_runner.py b/contrib/python/simplejson/py2/simplejson/tests/_cibw_runner.py deleted file mode 100644 index bffb70b8add..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/_cibw_runner.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Internal module for running tests from cibuildwheel""" - -import sys -import simplejson.tests - -if __name__ == '__main__': - simplejson.tests.main(project_dir=sys.argv[1]) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_bigint_as_string.py b/contrib/python/simplejson/py2/simplejson/tests/test_bigint_as_string.py deleted file mode 100644 index 2cf2cc2966e..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_bigint_as_string.py +++ /dev/null @@ -1,67 +0,0 @@ -from unittest import TestCase - -import simplejson as json - - -class TestBigintAsString(TestCase): - # Python 2.5, at least the one that ships on Mac OS X, calculates - # 2 ** 53 as 0! It manages to calculate 1 << 53 correctly. - values = [(200, 200), - ((1 << 53) - 1, 9007199254740991), - ((1 << 53), '9007199254740992'), - ((1 << 53) + 1, '9007199254740993'), - (-100, -100), - ((-1 << 53), '-9007199254740992'), - ((-1 << 53) - 1, '-9007199254740993'), - ((-1 << 53) + 1, -9007199254740991)] - - options = ( - {"bigint_as_string": True}, - {"int_as_string_bitcount": 53} - ) - - def test_ints(self): - for opts in self.options: - for val, expect in self.values: - self.assertEqual( - val, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, **opts))) - - def test_lists(self): - for opts in self.options: - for val, expect in self.values: - val = [val, val] - expect = [expect, expect] - self.assertEqual( - val, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, **opts))) - - def test_dicts(self): - for opts in self.options: - for val, expect in self.values: - val = {'k': val} - expect = {'k': expect} - self.assertEqual( - val, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, **opts))) - - def test_dict_keys(self): - for opts in self.options: - for val, _ in self.values: - expect = {str(val): 'value'} - val = {val: 'value'} - self.assertEqual( - expect, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, **opts))) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_bitsize_int_as_string.py b/contrib/python/simplejson/py2/simplejson/tests/test_bitsize_int_as_string.py deleted file mode 100644 index fd7d103a6d8..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_bitsize_int_as_string.py +++ /dev/null @@ -1,73 +0,0 @@ -from unittest import TestCase - -import simplejson as json - - -class TestBitSizeIntAsString(TestCase): - # Python 2.5, at least the one that ships on Mac OS X, calculates - # 2 ** 31 as 0! It manages to calculate 1 << 31 correctly. - values = [ - (200, 200), - ((1 << 31) - 1, (1 << 31) - 1), - ((1 << 31), str(1 << 31)), - ((1 << 31) + 1, str((1 << 31) + 1)), - (-100, -100), - ((-1 << 31), str(-1 << 31)), - ((-1 << 31) - 1, str((-1 << 31) - 1)), - ((-1 << 31) + 1, (-1 << 31) + 1), - ] - - def test_invalid_counts(self): - for n in ['foo', -1, 0, 1.0]: - self.assertRaises( - TypeError, - json.dumps, 0, int_as_string_bitcount=n) - - def test_ints_outside_range_fails(self): - self.assertNotEqual( - str(1 << 15), - json.loads(json.dumps(1 << 15, int_as_string_bitcount=16)), - ) - - def test_ints(self): - for val, expect in self.values: - self.assertEqual( - val, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, int_as_string_bitcount=31)), - ) - - def test_lists(self): - for val, expect in self.values: - val = [val, val] - expect = [expect, expect] - self.assertEqual( - val, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, int_as_string_bitcount=31))) - - def test_dicts(self): - for val, expect in self.values: - val = {'k': val} - expect = {'k': expect} - self.assertEqual( - val, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, int_as_string_bitcount=31))) - - def test_dict_keys(self): - for val, _ in self.values: - expect = {str(val): 'value'} - val = {val: 'value'} - self.assertEqual( - expect, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, int_as_string_bitcount=31))) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_check_circular.py b/contrib/python/simplejson/py2/simplejson/tests/test_check_circular.py deleted file mode 100644 index af6463d6d88..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_check_circular.py +++ /dev/null @@ -1,30 +0,0 @@ -from unittest import TestCase -import simplejson as json - -def default_iterable(obj): - return list(obj) - -class TestCheckCircular(TestCase): - def test_circular_dict(self): - dct = {} - dct['a'] = dct - self.assertRaises(ValueError, json.dumps, dct) - - def test_circular_list(self): - lst = [] - lst.append(lst) - self.assertRaises(ValueError, json.dumps, lst) - - def test_circular_composite(self): - dct2 = {} - dct2['a'] = [] - dct2['a'].append(dct2) - self.assertRaises(ValueError, json.dumps, dct2) - - def test_circular_default(self): - json.dumps([set()], default=default_iterable) - self.assertRaises(TypeError, json.dumps, [set()]) - - def test_circular_off_default(self): - json.dumps([set()], default=default_iterable, check_circular=False) - self.assertRaises(TypeError, json.dumps, [set()], check_circular=False) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_decimal.py b/contrib/python/simplejson/py2/simplejson/tests/test_decimal.py deleted file mode 100644 index 2b0940b1527..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_decimal.py +++ /dev/null @@ -1,71 +0,0 @@ -import decimal -from decimal import Decimal -from unittest import TestCase -from simplejson.compat import StringIO, reload_module - -import simplejson as json - -class TestDecimal(TestCase): - NUMS = "1.0", "10.00", "1.1", "1234567890.1234567890", "500" - def dumps(self, obj, **kw): - sio = StringIO() - json.dump(obj, sio, **kw) - res = json.dumps(obj, **kw) - self.assertEqual(res, sio.getvalue()) - return res - - def loads(self, s, **kw): - sio = StringIO(s) - res = json.loads(s, **kw) - self.assertEqual(res, json.load(sio, **kw)) - return res - - def test_decimal_encode(self): - for d in map(Decimal, self.NUMS): - self.assertEqual(self.dumps(d, use_decimal=True), str(d)) - - def test_decimal_decode(self): - for s in self.NUMS: - self.assertEqual(self.loads(s, parse_float=Decimal), Decimal(s)) - - def test_stringify_key(self): - for d in map(Decimal, self.NUMS): - v = {d: d} - self.assertEqual( - self.loads( - self.dumps(v, use_decimal=True), parse_float=Decimal), - {str(d): d}) - - def test_decimal_roundtrip(self): - for d in map(Decimal, self.NUMS): - # The type might not be the same (int and Decimal) but they - # should still compare equal. - for v in [d, [d], {'': d}]: - self.assertEqual( - self.loads( - self.dumps(v, use_decimal=True), parse_float=Decimal), - v) - - def test_decimal_defaults(self): - d = Decimal('1.1') - # use_decimal=True is the default - self.assertRaises(TypeError, json.dumps, d, use_decimal=False) - self.assertEqual('1.1', json.dumps(d)) - self.assertEqual('1.1', json.dumps(d, use_decimal=True)) - self.assertRaises(TypeError, json.dump, d, StringIO(), - use_decimal=False) - sio = StringIO() - json.dump(d, sio) - self.assertEqual('1.1', sio.getvalue()) - sio = StringIO() - json.dump(d, sio, use_decimal=True) - self.assertEqual('1.1', sio.getvalue()) - - def test_decimal_reload(self): - # Simulate a subinterpreter that reloads the Python modules but not - # the C code https://github.com/simplejson/simplejson/issues/34 - global Decimal - Decimal = reload_module(decimal).Decimal - import simplejson.encoder - simplejson.encoder.Decimal = Decimal - self.test_decimal_roundtrip() diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_decode.py b/contrib/python/simplejson/py2/simplejson/tests/test_decode.py deleted file mode 100644 index 317b4f98d55..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_decode.py +++ /dev/null @@ -1,127 +0,0 @@ -from __future__ import absolute_import -import decimal -from unittest import TestCase - -import sys -import simplejson as json -from simplejson.compat import StringIO, b, binary_type -from simplejson import OrderedDict - -class MisbehavingBytesSubtype(binary_type): - def decode(self, encoding=None): - return "bad decode" - def __str__(self): - return "bad __str__" - def __bytes__(self): - return b("bad __bytes__") - -class TestDecode(TestCase): - if not hasattr(TestCase, 'assertIs'): - def assertIs(self, a, b): - self.assertTrue(a is b, '%r is %r' % (a, b)) - - def test_decimal(self): - rval = json.loads('1.1', parse_float=decimal.Decimal) - self.assertTrue(isinstance(rval, decimal.Decimal)) - self.assertEqual(rval, decimal.Decimal('1.1')) - - def test_float(self): - rval = json.loads('1', parse_int=float) - self.assertTrue(isinstance(rval, float)) - self.assertEqual(rval, 1.0) - - def test_decoder_optimizations(self): - # Several optimizations were made that skip over calls to - # the whitespace regex, so this test is designed to try and - # exercise the uncommon cases. The array cases are already covered. - rval = json.loads('{ "key" : "value" , "k":"v" }') - self.assertEqual(rval, {"key":"value", "k":"v"}) - - def test_empty_objects(self): - s = '{}' - self.assertEqual(json.loads(s), eval(s)) - s = '[]' - self.assertEqual(json.loads(s), eval(s)) - s = '""' - self.assertEqual(json.loads(s), eval(s)) - - def test_object_pairs_hook(self): - s = '{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}' - p = [("xkd", 1), ("kcw", 2), ("art", 3), ("hxm", 4), - ("qrt", 5), ("pad", 6), ("hoy", 7)] - self.assertEqual(json.loads(s), eval(s)) - self.assertEqual(json.loads(s, object_pairs_hook=lambda x: x), p) - self.assertEqual(json.load(StringIO(s), - object_pairs_hook=lambda x: x), p) - od = json.loads(s, object_pairs_hook=OrderedDict) - self.assertEqual(od, OrderedDict(p)) - self.assertEqual(type(od), OrderedDict) - # the object_pairs_hook takes priority over the object_hook - self.assertEqual(json.loads(s, - object_pairs_hook=OrderedDict, - object_hook=lambda x: None), - OrderedDict(p)) - - def check_keys_reuse(self, source, loads): - rval = loads(source) - (a, b), (c, d) = sorted(rval[0]), sorted(rval[1]) - self.assertIs(a, c) - self.assertIs(b, d) - - def test_keys_reuse_str(self): - s = u'[{"a_key": 1, "b_\xe9": 2}, {"a_key": 3, "b_\xe9": 4}]'.encode('utf8') - self.check_keys_reuse(s, json.loads) - - def test_keys_reuse_unicode(self): - s = u'[{"a_key": 1, "b_\xe9": 2}, {"a_key": 3, "b_\xe9": 4}]' - self.check_keys_reuse(s, json.loads) - - def test_empty_strings(self): - self.assertEqual(json.loads('""'), "") - self.assertEqual(json.loads(u'""'), u"") - self.assertEqual(json.loads('[""]'), [""]) - self.assertEqual(json.loads(u'[""]'), [u""]) - - def test_raw_decode(self): - cls = json.decoder.JSONDecoder - self.assertEqual( - ({'a': {}}, 9), - cls().raw_decode("{\"a\": {}}")) - # http://code.google.com/p/simplejson/issues/detail?id=85 - self.assertEqual( - ({'a': {}}, 9), - cls(object_pairs_hook=dict).raw_decode("{\"a\": {}}")) - # https://github.com/simplejson/simplejson/pull/38 - self.assertEqual( - ({'a': {}}, 11), - cls().raw_decode(" \n{\"a\": {}}")) - - def test_bytes_decode(self): - cls = json.decoder.JSONDecoder - data = b('"\xe2\x82\xac"') - self.assertEqual(cls().decode(data), u'\u20ac') - self.assertEqual(cls(encoding='latin1').decode(data), u'\xe2\x82\xac') - self.assertEqual(cls(encoding=None).decode(data), u'\u20ac') - - data = MisbehavingBytesSubtype(b('"\xe2\x82\xac"')) - self.assertEqual(cls().decode(data), u'\u20ac') - self.assertEqual(cls(encoding='latin1').decode(data), u'\xe2\x82\xac') - self.assertEqual(cls(encoding=None).decode(data), u'\u20ac') - - def test_bounds_checking(self): - # https://github.com/simplejson/simplejson/issues/98 - j = json.decoder.JSONDecoder() - for i in [4, 5, 6, -1, -2, -3, -4, -5, -6]: - self.assertRaises(ValueError, j.scan_once, '1234', i) - self.assertRaises(ValueError, j.raw_decode, '1234', i) - x, y = sorted(['128931233', '472389423'], key=id) - diff = id(x) - id(y) - self.assertRaises(ValueError, j.scan_once, y, diff) - self.assertRaises(ValueError, j.raw_decode, y, i) - - def test_bounded_int(self): - # SJ-PT-23-03, limit quadratic number parsing per Python 3.11 - max_str_digits = getattr(sys, 'get_int_max_str_digits', lambda: 4300)() - s = '1' + '0' * (max_str_digits - 1) - self.assertEqual(json.loads(s), int(s)) - self.assertRaises(ValueError, json.loads, s + '0') diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_default.py b/contrib/python/simplejson/py2/simplejson/tests/test_default.py deleted file mode 100644 index d1eacb8c4c3..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_default.py +++ /dev/null @@ -1,9 +0,0 @@ -from unittest import TestCase - -import simplejson as json - -class TestDefault(TestCase): - def test_default(self): - self.assertEqual( - json.dumps(type, default=repr), - json.dumps(repr(type))) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_dump.py b/contrib/python/simplejson/py2/simplejson/tests/test_dump.py deleted file mode 100644 index 5628489f268..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_dump.py +++ /dev/null @@ -1,255 +0,0 @@ -from unittest import TestCase -from simplejson.compat import StringIO, long_type, b, binary_type, text_type, PY3 -import simplejson as json - -class MisbehavingTextSubtype(text_type): - def __str__(self): - return "FAIL!" - -class MisbehavingBytesSubtype(binary_type): - def decode(self, encoding=None): - return "bad decode" - def __str__(self): - return "bad __str__" - def __bytes__(self): - return b("bad __bytes__") - -def as_text_type(s): - if PY3 and isinstance(s, bytes): - return s.decode('ascii') - return s - -def decode_iso_8859_15(b): - return b.decode('iso-8859-15') - -class TestDump(TestCase): - def test_dump(self): - sio = StringIO() - json.dump({}, sio) - self.assertEqual(sio.getvalue(), '{}') - - def test_constants(self): - for c in [None, True, False]: - self.assertTrue(json.loads(json.dumps(c)) is c) - self.assertTrue(json.loads(json.dumps([c]))[0] is c) - self.assertTrue(json.loads(json.dumps({'a': c}))['a'] is c) - - def test_stringify_key(self): - items = [(b('bytes'), 'bytes'), - (1.0, '1.0'), - (10, '10'), - (True, 'true'), - (False, 'false'), - (None, 'null'), - (long_type(100), '100')] - for k, expect in items: - self.assertEqual( - json.loads(json.dumps({k: expect})), - {expect: expect}) - self.assertEqual( - json.loads(json.dumps({k: expect}, sort_keys=True)), - {expect: expect}) - self.assertRaises(TypeError, json.dumps, {json: 1}) - for v in [{}, {'other': 1}, {b('derp'): 1, 'herp': 2}]: - for sort_keys in [False, True]: - v0 = dict(v) - v0[json] = 1 - v1 = dict((as_text_type(key), val) for (key, val) in v.items()) - self.assertEqual( - json.loads(json.dumps(v0, skipkeys=True, sort_keys=sort_keys)), - v1) - self.assertEqual( - json.loads(json.dumps({'': v0}, skipkeys=True, sort_keys=sort_keys)), - {'': v1}) - self.assertEqual( - json.loads(json.dumps([v0], skipkeys=True, sort_keys=sort_keys)), - [v1]) - - def test_dumps(self): - self.assertEqual(json.dumps({}), '{}') - - def test_encode_truefalse(self): - self.assertEqual(json.dumps( - {True: False, False: True}, sort_keys=True), - '{"false": true, "true": false}') - self.assertEqual( - # load first because the keys are not sorted - json.loads(json.dumps({'k1': {False: 5}, 'k2': {0: 5}})), - {'k1': {'false': 5}, 'k2': {'0': 5}}, - ) - - self.assertEqual( - json.dumps( - {2: 3.0, - 4.0: long_type(5), - False: 1, - long_type(6): True, - "7": 0}, - sort_keys=True), - '{"2": 3.0, "4.0": 5, "6": true, "7": 0, "false": 1}') - - def test_ordered_dict(self): - # http://bugs.python.org/issue6105 - items = [('one', 1), ('two', 2), ('three', 3), ('four', 4), ('five', 5)] - s = json.dumps(json.OrderedDict(items)) - self.assertEqual( - s, - '{"one": 1, "two": 2, "three": 3, "four": 4, "five": 5}') - - def test_indent_unknown_type_acceptance(self): - """ - A test against the regression mentioned at `github issue 29`_. - - The indent parameter should accept any type which pretends to be - an instance of int or long when it comes to being multiplied by - strings, even if it is not actually an int or long, for - backwards compatibility. - - .. _github issue 29: - http://github.com/simplejson/simplejson/issue/29 - """ - - class AwesomeInt(object): - """An awesome reimplementation of integers""" - - def __init__(self, *args, **kwargs): - if len(args) > 0: - # [construct from literals, objects, etc.] - # ... - - # Finally, if args[0] is an integer, store it - if isinstance(args[0], int): - self._int = args[0] - - # [various methods] - - def __mul__(self, other): - # [various ways to multiply AwesomeInt objects] - # ... finally, if the right-hand operand is not awesome enough, - # try to do a normal integer multiplication - if hasattr(self, '_int'): - return self._int * other - else: - raise NotImplementedError("To do non-awesome things with" - " this object, please construct it from an integer!") - - s = json.dumps([0, 1, 2], indent=AwesomeInt(3)) - self.assertEqual(s, '[\n 0,\n 1,\n 2\n]') - - def test_accumulator(self): - # the C API uses an accumulator that collects after 100,000 appends - lst = [0] * 100000 - self.assertEqual(json.loads(json.dumps(lst)), lst) - - def test_sort_keys(self): - # https://github.com/simplejson/simplejson/issues/106 - for num_keys in range(2, 32): - p = dict((str(x), x) for x in range(num_keys)) - sio = StringIO() - json.dump(p, sio, sort_keys=True) - self.assertEqual(sio.getvalue(), json.dumps(p, sort_keys=True)) - self.assertEqual(json.loads(sio.getvalue()), p) - - def test_misbehaving_text_subtype(self): - # https://github.com/simplejson/simplejson/issues/185 - text = "this is some text" - self.assertEqual( - json.dumps(MisbehavingTextSubtype(text)), - json.dumps(text) - ) - self.assertEqual( - json.dumps([MisbehavingTextSubtype(text)]), - json.dumps([text]) - ) - self.assertEqual( - json.dumps({MisbehavingTextSubtype(text): 42}), - json.dumps({text: 42}) - ) - - def test_misbehaving_bytes_subtype(self): - data = b("this is some data \xe2\x82\xac") - self.assertEqual( - json.dumps(MisbehavingBytesSubtype(data)), - json.dumps(data) - ) - self.assertEqual( - json.dumps([MisbehavingBytesSubtype(data)]), - json.dumps([data]) - ) - self.assertEqual( - json.dumps({MisbehavingBytesSubtype(data): 42}), - json.dumps({data: 42}) - ) - - def test_bytes_toplevel(self): - self.assertEqual(json.dumps(b('\xe2\x82\xac')), r'"\u20ac"') - self.assertRaises(UnicodeDecodeError, json.dumps, b('\xa4')) - self.assertEqual(json.dumps(b('\xa4'), encoding='iso-8859-1'), - r'"\u00a4"') - self.assertEqual(json.dumps(b('\xa4'), encoding='iso-8859-15'), - r'"\u20ac"') - if PY3: - self.assertRaises(TypeError, json.dumps, b('\xe2\x82\xac'), - encoding=None) - self.assertRaises(TypeError, json.dumps, b('\xa4'), - encoding=None) - self.assertEqual(json.dumps(b('\xa4'), encoding=None, - default=decode_iso_8859_15), - r'"\u20ac"') - else: - self.assertEqual(json.dumps(b('\xe2\x82\xac'), encoding=None), - r'"\u20ac"') - self.assertRaises(UnicodeDecodeError, json.dumps, b('\xa4'), - encoding=None) - self.assertRaises(UnicodeDecodeError, json.dumps, b('\xa4'), - encoding=None, default=decode_iso_8859_15) - - def test_bytes_nested(self): - self.assertEqual(json.dumps([b('\xe2\x82\xac')]), r'["\u20ac"]') - self.assertRaises(UnicodeDecodeError, json.dumps, [b('\xa4')]) - self.assertEqual(json.dumps([b('\xa4')], encoding='iso-8859-1'), - r'["\u00a4"]') - self.assertEqual(json.dumps([b('\xa4')], encoding='iso-8859-15'), - r'["\u20ac"]') - if PY3: - self.assertRaises(TypeError, json.dumps, [b('\xe2\x82\xac')], - encoding=None) - self.assertRaises(TypeError, json.dumps, [b('\xa4')], - encoding=None) - self.assertEqual(json.dumps([b('\xa4')], encoding=None, - default=decode_iso_8859_15), - r'["\u20ac"]') - else: - self.assertEqual(json.dumps([b('\xe2\x82\xac')], encoding=None), - r'["\u20ac"]') - self.assertRaises(UnicodeDecodeError, json.dumps, [b('\xa4')], - encoding=None) - self.assertRaises(UnicodeDecodeError, json.dumps, [b('\xa4')], - encoding=None, default=decode_iso_8859_15) - - def test_bytes_key(self): - self.assertEqual(json.dumps({b('\xe2\x82\xac'): 42}), r'{"\u20ac": 42}') - self.assertRaises(UnicodeDecodeError, json.dumps, {b('\xa4'): 42}) - self.assertEqual(json.dumps({b('\xa4'): 42}, encoding='iso-8859-1'), - r'{"\u00a4": 42}') - self.assertEqual(json.dumps({b('\xa4'): 42}, encoding='iso-8859-15'), - r'{"\u20ac": 42}') - if PY3: - self.assertRaises(TypeError, json.dumps, {b('\xe2\x82\xac'): 42}, - encoding=None) - self.assertRaises(TypeError, json.dumps, {b('\xa4'): 42}, - encoding=None) - self.assertRaises(TypeError, json.dumps, {b('\xa4'): 42}, - encoding=None, default=decode_iso_8859_15) - self.assertEqual(json.dumps({b('\xa4'): 42}, encoding=None, - skipkeys=True), - r'{}') - else: - self.assertEqual(json.dumps({b('\xe2\x82\xac'): 42}, encoding=None), - r'{"\u20ac": 42}') - self.assertRaises(UnicodeDecodeError, json.dumps, {b('\xa4'): 42}, - encoding=None) - self.assertRaises(UnicodeDecodeError, json.dumps, {b('\xa4'): 42}, - encoding=None, default=decode_iso_8859_15) - self.assertRaises(UnicodeDecodeError, json.dumps, {b('\xa4'): 42}, - encoding=None, skipkeys=True) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_encode_basestring_ascii.py b/contrib/python/simplejson/py2/simplejson/tests/test_encode_basestring_ascii.py deleted file mode 100644 index 49706bfd978..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_encode_basestring_ascii.py +++ /dev/null @@ -1,47 +0,0 @@ -from unittest import TestCase - -import simplejson.encoder -from simplejson.compat import b - -CASES = [ - (u'/\\"\ucafe\ubabe\uab98\ufcde\ubcda\uef4a\x08\x0c\n\r\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?', '"/\\\\\\"\\ucafe\\ubabe\\uab98\\ufcde\\ubcda\\uef4a\\b\\f\\n\\r\\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?"'), - (u'\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), - (u'controls', '"controls"'), - (u'\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'), - (u'{"object with 1 member":["array with 1 element"]}', '"{\\"object with 1 member\\":[\\"array with 1 element\\"]}"'), - (u' s p a c e d ', '" s p a c e d "'), - (u'\U0001d120', '"\\ud834\\udd20"'), - (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'), - (b('\xce\xb1\xce\xa9'), '"\\u03b1\\u03a9"'), - (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'), - (b('\xce\xb1\xce\xa9'), '"\\u03b1\\u03a9"'), - (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'), - (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'), - (u"`1~!@#$%^&*()_+-={':[,]}|;.</>?", '"`1~!@#$%^&*()_+-={\':[,]}|;.</>?"'), - (u'\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'), - (u'\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), -] - -class TestEncodeBaseStringAscii(TestCase): - def test_py_encode_basestring_ascii(self): - self._test_encode_basestring_ascii(simplejson.encoder.py_encode_basestring_ascii) - - def test_c_encode_basestring_ascii(self): - if not simplejson.encoder.c_encode_basestring_ascii: - return - self._test_encode_basestring_ascii(simplejson.encoder.c_encode_basestring_ascii) - - def _test_encode_basestring_ascii(self, encode_basestring_ascii): - fname = encode_basestring_ascii.__name__ - for input_string, expect in CASES: - result = encode_basestring_ascii(input_string) - #self.assertEqual(result, expect, - # '{0!r} != {1!r} for {2}({3!r})'.format( - # result, expect, fname, input_string)) - self.assertEqual(result, expect, - '%r != %r for %s(%r)' % (result, expect, fname, input_string)) - - def test_sorted_dict(self): - items = [('one', 1), ('two', 2), ('three', 3), ('four', 4), ('five', 5)] - s = simplejson.dumps(dict(items), sort_keys=True) - self.assertEqual(s, '{"five": 5, "four": 4, "one": 1, "three": 3, "two": 2}') diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_encode_for_html.py b/contrib/python/simplejson/py2/simplejson/tests/test_encode_for_html.py deleted file mode 100644 index 3a840aa48cb..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_encode_for_html.py +++ /dev/null @@ -1,38 +0,0 @@ -import unittest - -import simplejson as json - -class TestEncodeForHTML(unittest.TestCase): - - def setUp(self): - self.decoder = json.JSONDecoder() - self.encoder = json.JSONEncoderForHTML() - self.non_ascii_encoder = json.JSONEncoderForHTML(ensure_ascii=False) - - def test_basic_encode(self): - self.assertEqual(r'"\u0026"', self.encoder.encode('&')) - self.assertEqual(r'"\u003c"', self.encoder.encode('<')) - self.assertEqual(r'"\u003e"', self.encoder.encode('>')) - self.assertEqual(r'"\u2028"', self.encoder.encode(u'\u2028')) - - def test_non_ascii_basic_encode(self): - self.assertEqual(r'"\u0026"', self.non_ascii_encoder.encode('&')) - self.assertEqual(r'"\u003c"', self.non_ascii_encoder.encode('<')) - self.assertEqual(r'"\u003e"', self.non_ascii_encoder.encode('>')) - self.assertEqual(r'"\u2028"', self.non_ascii_encoder.encode(u'\u2028')) - - def test_basic_roundtrip(self): - for char in '&<>': - self.assertEqual( - char, self.decoder.decode( - self.encoder.encode(char))) - - def test_prevent_script_breakout(self): - bad_string = '</script><script>alert("gotcha")</script>' - self.assertEqual( - r'"\u003c/script\u003e\u003cscript\u003e' - r'alert(\"gotcha\")\u003c/script\u003e"', - self.encoder.encode(bad_string)) - self.assertEqual( - bad_string, self.decoder.decode( - self.encoder.encode(bad_string))) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_errors.py b/contrib/python/simplejson/py2/simplejson/tests/test_errors.py deleted file mode 100644 index d5738254541..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_errors.py +++ /dev/null @@ -1,68 +0,0 @@ -import sys, pickle -from unittest import TestCase - -import simplejson as json -from simplejson.compat import text_type, b - -class TestErrors(TestCase): - def test_string_keys_error(self): - data = [{'a': 'A', 'b': (2, 4), 'c': 3.0, ('d',): 'D tuple'}] - try: - json.dumps(data) - except TypeError: - err = sys.exc_info()[1] - else: - self.fail('Expected TypeError') - self.assertEqual(str(err), - 'keys must be str, int, float, bool or None, not tuple') - - def test_not_serializable(self): - try: - json.dumps(json) - except TypeError: - err = sys.exc_info()[1] - else: - self.fail('Expected TypeError') - self.assertEqual(str(err), - 'Object of type module is not JSON serializable') - - def test_decode_error(self): - err = None - try: - json.loads('{}\na\nb') - except json.JSONDecodeError: - err = sys.exc_info()[1] - else: - self.fail('Expected JSONDecodeError') - self.assertEqual(err.lineno, 2) - self.assertEqual(err.colno, 1) - self.assertEqual(err.endlineno, 3) - self.assertEqual(err.endcolno, 2) - - def test_scan_error(self): - err = None - for t in (text_type, b): - try: - json.loads(t('{"asdf": "')) - except json.JSONDecodeError: - err = sys.exc_info()[1] - else: - self.fail('Expected JSONDecodeError') - self.assertEqual(err.lineno, 1) - self.assertEqual(err.colno, 10) - - def test_error_is_pickable(self): - err = None - try: - json.loads('{}\na\nb') - except json.JSONDecodeError: - err = sys.exc_info()[1] - else: - self.fail('Expected JSONDecodeError') - s = pickle.dumps(err) - e = pickle.loads(s) - - self.assertEqual(err.msg, e.msg) - self.assertEqual(err.doc, e.doc) - self.assertEqual(err.pos, e.pos) - self.assertEqual(err.end, e.end) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_fail.py b/contrib/python/simplejson/py2/simplejson/tests/test_fail.py deleted file mode 100644 index 5f9a8f698cf..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_fail.py +++ /dev/null @@ -1,178 +0,0 @@ -import sys -from unittest import TestCase - -import simplejson as json - -# 2007-10-05 -JSONDOCS = [ - # http://json.org/JSON_checker/test/fail1.json - '"A JSON payload should be an object or array, not a string."', - # http://json.org/JSON_checker/test/fail2.json - '["Unclosed array"', - # http://json.org/JSON_checker/test/fail3.json - '{unquoted_key: "keys must be quoted"}', - # http://json.org/JSON_checker/test/fail4.json - '["extra comma",]', - # http://json.org/JSON_checker/test/fail5.json - '["double extra comma",,]', - # http://json.org/JSON_checker/test/fail6.json - '[ , "<-- missing value"]', - # http://json.org/JSON_checker/test/fail7.json - '["Comma after the close"],', - # http://json.org/JSON_checker/test/fail8.json - '["Extra close"]]', - # http://json.org/JSON_checker/test/fail9.json - '{"Extra comma": true,}', - # http://json.org/JSON_checker/test/fail10.json - '{"Extra value after close": true} "misplaced quoted value"', - # http://json.org/JSON_checker/test/fail11.json - '{"Illegal expression": 1 + 2}', - # http://json.org/JSON_checker/test/fail12.json - '{"Illegal invocation": alert()}', - # http://json.org/JSON_checker/test/fail13.json - '{"Numbers cannot have leading zeroes": 013}', - # http://json.org/JSON_checker/test/fail14.json - '{"Numbers cannot be hex": 0x14}', - # http://json.org/JSON_checker/test/fail15.json - '["Illegal backslash escape: \\x15"]', - # http://json.org/JSON_checker/test/fail16.json - '[\\naked]', - # http://json.org/JSON_checker/test/fail17.json - '["Illegal backslash escape: \\017"]', - # http://json.org/JSON_checker/test/fail18.json - '[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]', - # http://json.org/JSON_checker/test/fail19.json - '{"Missing colon" null}', - # http://json.org/JSON_checker/test/fail20.json - '{"Double colon":: null}', - # http://json.org/JSON_checker/test/fail21.json - '{"Comma instead of colon", null}', - # http://json.org/JSON_checker/test/fail22.json - '["Colon instead of comma": false]', - # http://json.org/JSON_checker/test/fail23.json - '["Bad value", truth]', - # http://json.org/JSON_checker/test/fail24.json - "['single quote']", - # http://json.org/JSON_checker/test/fail25.json - '["\ttab\tcharacter\tin\tstring\t"]', - # http://json.org/JSON_checker/test/fail26.json - '["tab\\ character\\ in\\ string\\ "]', - # http://json.org/JSON_checker/test/fail27.json - '["line\nbreak"]', - # http://json.org/JSON_checker/test/fail28.json - '["line\\\nbreak"]', - # http://json.org/JSON_checker/test/fail29.json - '[0e]', - # http://json.org/JSON_checker/test/fail30.json - '[0e+]', - # http://json.org/JSON_checker/test/fail31.json - '[0e+-1]', - # http://json.org/JSON_checker/test/fail32.json - '{"Comma instead if closing brace": true,', - # http://json.org/JSON_checker/test/fail33.json - '["mismatch"}', - # http://code.google.com/p/simplejson/issues/detail?id=3 - u'["A\u001FZ control characters in string"]', - # misc based on coverage - '{', - '{]', - '{"foo": "bar"]', - '{"foo": "bar"', - 'nul', - 'nulx', - '-', - '-x', - '-e', - '-e0', - '-Infinite', - '-Inf', - 'Infinit', - 'Infinite', - 'NaM', - 'NuN', - 'falsy', - 'fal', - 'trug', - 'tru', - '1e', - '1ex', - '1e-', - '1e-x', -] - -SKIPS = { - 1: "why not have a string payload?", - 18: "spec doesn't specify any nesting limitations", -} - -class TestFail(TestCase): - def test_failures(self): - for idx, doc in enumerate(JSONDOCS): - idx = idx + 1 - if idx in SKIPS: - json.loads(doc) - continue - try: - json.loads(doc) - except json.JSONDecodeError: - pass - else: - self.fail("Expected failure for fail%d.json: %r" % (idx, doc)) - - def test_array_decoder_issue46(self): - # http://code.google.com/p/simplejson/issues/detail?id=46 - for doc in [u'[,]', '[,]']: - try: - json.loads(doc) - except json.JSONDecodeError: - e = sys.exc_info()[1] - self.assertEqual(e.pos, 1) - self.assertEqual(e.lineno, 1) - self.assertEqual(e.colno, 2) - except Exception: - e = sys.exc_info()[1] - self.fail("Unexpected exception raised %r %s" % (e, e)) - else: - self.fail("Unexpected success parsing '[,]'") - - def test_truncated_input(self): - test_cases = [ - ('', 'Expecting value', 0), - ('[', "Expecting value or ']'", 1), - ('[42', "Expecting ',' delimiter", 3), - ('[42,', 'Expecting value', 4), - ('["', 'Unterminated string starting at', 1), - ('["spam', 'Unterminated string starting at', 1), - ('["spam"', "Expecting ',' delimiter", 7), - ('["spam",', 'Expecting value', 8), - ('{', "Expecting property name enclosed in double quotes or '}'", 1), - ('{"', 'Unterminated string starting at', 1), - ('{"spam', 'Unterminated string starting at', 1), - ('{"spam"', "Expecting ':' delimiter", 7), - ('{"spam":', 'Expecting value', 8), - ('{"spam":42', "Expecting ',' delimiter", 10), - ('{"spam":42,', 'Expecting property name enclosed in double quotes', - 11), - ('"', 'Unterminated string starting at', 0), - ('"spam', 'Unterminated string starting at', 0), - ('[,', "Expecting value", 1), - ('--', 'Expecting value', 0), - ('"\x18d', "Invalid control character %r", 1), - ] - for data, msg, idx in test_cases: - try: - json.loads(data) - except json.JSONDecodeError: - e = sys.exc_info()[1] - self.assertEqual( - e.msg[:len(msg)], - msg, - "%r doesn't start with %r for %r" % (e.msg, msg, data)) - self.assertEqual( - e.pos, idx, - "pos %r != %r for %r" % (e.pos, idx, data)) - except Exception: - e = sys.exc_info()[1] - self.fail("Unexpected exception raised %r %s" % (e, e)) - else: - self.fail("Unexpected success parsing '%r'" % (data,)) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_float.py b/contrib/python/simplejson/py2/simplejson/tests/test_float.py deleted file mode 100644 index fed1bd97cd6..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_float.py +++ /dev/null @@ -1,46 +0,0 @@ -import sys -import math -from unittest import TestCase -from simplejson.compat import long_type, text_type -import simplejson as json -from simplejson.decoder import NaN, PosInf, NegInf - -class TestFloat(TestCase): - def test_degenerates_allow(self): - for inf in (PosInf, NegInf): - self.assertEqual(json.loads(json.dumps(inf, allow_nan=True), allow_nan=True), inf) - # Python 2.5 doesn't have math.isnan - nan = json.loads(json.dumps(NaN, allow_nan=True), allow_nan=True) - self.assertTrue((0 + nan) != nan) - - def test_degenerates_ignore(self): - for f in (PosInf, NegInf, NaN): - self.assertEqual(json.loads(json.dumps(f, ignore_nan=True)), None) - - def test_degenerates_deny(self): - for f in (PosInf, NegInf, NaN): - self.assertRaises(ValueError, json.dumps, f, allow_nan=False) - for s in ('Infinity', '-Infinity', 'NaN'): - self.assertRaises(ValueError, json.loads, s, allow_nan=False) - self.assertRaises(ValueError, json.loads, s) - - def test_floats(self): - for num in [1617161771.7650001, math.pi, math.pi**100, - math.pi**-100, 3.1]: - self.assertEqual(float(json.dumps(num)), num) - self.assertEqual(json.loads(json.dumps(num)), num) - self.assertEqual(json.loads(text_type(json.dumps(num))), num) - - def test_ints(self): - for num in [1, long_type(1), 1<<32, 1<<64]: - self.assertEqual(json.dumps(num), str(num)) - self.assertEqual(int(json.dumps(num)), num) - self.assertEqual(json.loads(json.dumps(num)), num) - self.assertEqual(json.loads(text_type(json.dumps(num))), num) - - def test_float_range(self): - try: - float_range = [sys.float_info.min, sys.float_info.max] - except AttributeError: - float_range = [2.2250738585072014e-308, 1.7976931348623157e+308] - self.assertEqual(json.loads(json.dumps(float_range)), float_range) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_for_json.py b/contrib/python/simplejson/py2/simplejson/tests/test_for_json.py deleted file mode 100644 index b791b883b04..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_for_json.py +++ /dev/null @@ -1,97 +0,0 @@ -import unittest -import simplejson as json - - -class ForJson(object): - def for_json(self): - return {'for_json': 1} - - -class NestedForJson(object): - def for_json(self): - return {'nested': ForJson()} - - -class ForJsonList(object): - def for_json(self): - return ['list'] - - -class DictForJson(dict): - def for_json(self): - return {'alpha': 1} - - -class ListForJson(list): - def for_json(self): - return ['list'] - - -class TestForJson(unittest.TestCase): - def assertRoundTrip(self, obj, other, for_json=True): - if for_json is None: - # None will use the default - s = json.dumps(obj) - else: - s = json.dumps(obj, for_json=for_json) - self.assertEqual( - json.loads(s), - other) - - def test_for_json_encodes_stand_alone_object(self): - self.assertRoundTrip( - ForJson(), - ForJson().for_json()) - - def test_for_json_encodes_object_nested_in_dict(self): - self.assertRoundTrip( - {'hooray': ForJson()}, - {'hooray': ForJson().for_json()}) - - def test_for_json_encodes_object_nested_in_list_within_dict(self): - self.assertRoundTrip( - {'list': [0, ForJson(), 2, 3]}, - {'list': [0, ForJson().for_json(), 2, 3]}) - - def test_for_json_encodes_object_nested_within_object(self): - self.assertRoundTrip( - NestedForJson(), - {'nested': {'for_json': 1}}) - - def test_for_json_encodes_list(self): - self.assertRoundTrip( - ForJsonList(), - ForJsonList().for_json()) - - def test_for_json_encodes_list_within_object(self): - self.assertRoundTrip( - {'nested': ForJsonList()}, - {'nested': ForJsonList().for_json()}) - - def test_for_json_encodes_dict_subclass(self): - self.assertRoundTrip( - DictForJson(a=1), - DictForJson(a=1).for_json()) - - def test_for_json_encodes_list_subclass(self): - self.assertRoundTrip( - ListForJson(['l']), - ListForJson(['l']).for_json()) - - def test_for_json_ignored_if_not_true_with_dict_subclass(self): - for for_json in (None, False): - self.assertRoundTrip( - DictForJson(a=1), - {'a': 1}, - for_json=for_json) - - def test_for_json_ignored_if_not_true_with_list_subclass(self): - for for_json in (None, False): - self.assertRoundTrip( - ListForJson(['l']), - ['l'], - for_json=for_json) - - def test_raises_typeerror_if_for_json_not_true_with_object(self): - self.assertRaises(TypeError, json.dumps, ForJson()) - self.assertRaises(TypeError, json.dumps, ForJson(), for_json=False) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_indent.py b/contrib/python/simplejson/py2/simplejson/tests/test_indent.py deleted file mode 100644 index cea25a575ed..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_indent.py +++ /dev/null @@ -1,86 +0,0 @@ -from unittest import TestCase -import textwrap - -import simplejson as json -from simplejson.compat import StringIO - -class TestIndent(TestCase): - def test_indent(self): - h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', - 'i-vhbjkhnth', - {'nifty': 87}, {'field': 'yes', 'morefield': False} ] - - expect = textwrap.dedent("""\ - [ - \t[ - \t\t"blorpie" - \t], - \t[ - \t\t"whoops" - \t], - \t[], - \t"d-shtaeou", - \t"d-nthiouh", - \t"i-vhbjkhnth", - \t{ - \t\t"nifty": 87 - \t}, - \t{ - \t\t"field": "yes", - \t\t"morefield": false - \t} - ]""") - - - d1 = json.dumps(h) - d2 = json.dumps(h, indent='\t', sort_keys=True, separators=(',', ': ')) - d3 = json.dumps(h, indent=' ', sort_keys=True, separators=(',', ': ')) - d4 = json.dumps(h, indent=2, sort_keys=True, separators=(',', ': ')) - - h1 = json.loads(d1) - h2 = json.loads(d2) - h3 = json.loads(d3) - h4 = json.loads(d4) - - self.assertEqual(h1, h) - self.assertEqual(h2, h) - self.assertEqual(h3, h) - self.assertEqual(h4, h) - self.assertEqual(d3, expect.replace('\t', ' ')) - self.assertEqual(d4, expect.replace('\t', ' ')) - # NOTE: Python 2.4 textwrap.dedent converts tabs to spaces, - # so the following is expected to fail. Python 2.4 is not a - # supported platform in simplejson 2.1.0+. - self.assertEqual(d2, expect) - - def test_indent0(self): - h = {3: 1} - def check(indent, expected): - d1 = json.dumps(h, indent=indent) - self.assertEqual(d1, expected) - - sio = StringIO() - json.dump(h, sio, indent=indent) - self.assertEqual(sio.getvalue(), expected) - - # indent=0 should emit newlines - check(0, '{\n"3": 1\n}') - # indent=None is more compact - check(None, '{"3": 1}') - - def test_separators(self): - lst = [1,2,3,4] - expect = '[\n1,\n2,\n3,\n4\n]' - expect_spaces = '[\n1, \n2, \n3, \n4\n]' - # Ensure that separators still works - self.assertEqual( - expect_spaces, - json.dumps(lst, indent=0, separators=(', ', ': '))) - # Force the new defaults - self.assertEqual( - expect, - json.dumps(lst, indent=0, separators=(',', ': '))) - # Added in 2.1.4 - self.assertEqual( - expect, - json.dumps(lst, indent=0)) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_item_sort_key.py b/contrib/python/simplejson/py2/simplejson/tests/test_item_sort_key.py deleted file mode 100644 index 98971b86ac7..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_item_sort_key.py +++ /dev/null @@ -1,27 +0,0 @@ -from unittest import TestCase - -import simplejson as json -from operator import itemgetter - -class TestItemSortKey(TestCase): - def test_simple_first(self): - a = {'a': 1, 'c': 5, 'jack': 'jill', 'pick': 'axe', 'array': [1, 5, 6, 9], 'tuple': (83, 12, 3), 'crate': 'dog', 'zeak': 'oh'} - self.assertEqual( - '{"a": 1, "c": 5, "crate": "dog", "jack": "jill", "pick": "axe", "zeak": "oh", "array": [1, 5, 6, 9], "tuple": [83, 12, 3]}', - json.dumps(a, item_sort_key=json.simple_first)) - - def test_case(self): - a = {'a': 1, 'c': 5, 'Jack': 'jill', 'pick': 'axe', 'Array': [1, 5, 6, 9], 'tuple': (83, 12, 3), 'crate': 'dog', 'zeak': 'oh'} - self.assertEqual( - '{"Array": [1, 5, 6, 9], "Jack": "jill", "a": 1, "c": 5, "crate": "dog", "pick": "axe", "tuple": [83, 12, 3], "zeak": "oh"}', - json.dumps(a, item_sort_key=itemgetter(0))) - self.assertEqual( - '{"a": 1, "Array": [1, 5, 6, 9], "c": 5, "crate": "dog", "Jack": "jill", "pick": "axe", "tuple": [83, 12, 3], "zeak": "oh"}', - json.dumps(a, item_sort_key=lambda kv: kv[0].lower())) - - def test_item_sort_key_value(self): - # https://github.com/simplejson/simplejson/issues/173 - a = {'a': 1, 'b': 0} - self.assertEqual( - '{"b": 0, "a": 1}', - json.dumps(a, item_sort_key=lambda kv: kv[1])) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_iterable.py b/contrib/python/simplejson/py2/simplejson/tests/test_iterable.py deleted file mode 100644 index 35d3e759111..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_iterable.py +++ /dev/null @@ -1,31 +0,0 @@ -import unittest -from simplejson.compat import StringIO - -import simplejson as json - -def iter_dumps(obj, **kw): - return ''.join(json.JSONEncoder(**kw).iterencode(obj)) - -def sio_dump(obj, **kw): - sio = StringIO() - json.dumps(obj, **kw) - return sio.getvalue() - -class TestIterable(unittest.TestCase): - def test_iterable(self): - for l in ([], [1], [1, 2], [1, 2, 3]): - for opts in [{}, {'indent': 2}]: - for dumps in (json.dumps, iter_dumps, sio_dump): - expect = dumps(l, **opts) - default_expect = dumps(sum(l), **opts) - # Default is False - self.assertRaises(TypeError, dumps, iter(l), **opts) - self.assertRaises(TypeError, dumps, iter(l), iterable_as_array=False, **opts) - self.assertEqual(expect, dumps(iter(l), iterable_as_array=True, **opts)) - # Ensure that the "default" gets called - self.assertEqual(default_expect, dumps(iter(l), default=sum, **opts)) - self.assertEqual(default_expect, dumps(iter(l), iterable_as_array=False, default=sum, **opts)) - # Ensure that the "default" does not get called - self.assertEqual( - expect, - dumps(iter(l), iterable_as_array=True, default=sum, **opts)) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_namedtuple.py b/contrib/python/simplejson/py2/simplejson/tests/test_namedtuple.py deleted file mode 100644 index cc0f8aa1d7d..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_namedtuple.py +++ /dev/null @@ -1,174 +0,0 @@ -from __future__ import absolute_import -import unittest -import simplejson as json -from simplejson.compat import StringIO - -try: - from unittest import mock -except ImportError: - mock = None - -try: - from collections import namedtuple -except ImportError: - class Value(tuple): - def __new__(cls, *args): - return tuple.__new__(cls, args) - - def _asdict(self): - return {'value': self[0]} - class Point(tuple): - def __new__(cls, *args): - return tuple.__new__(cls, args) - - def _asdict(self): - return {'x': self[0], 'y': self[1]} -else: - Value = namedtuple('Value', ['value']) - Point = namedtuple('Point', ['x', 'y']) - -class DuckValue(object): - def __init__(self, *args): - self.value = Value(*args) - - def _asdict(self): - return self.value._asdict() - -class DuckPoint(object): - def __init__(self, *args): - self.point = Point(*args) - - def _asdict(self): - return self.point._asdict() - -class DeadDuck(object): - _asdict = None - -class DeadDict(dict): - _asdict = None - -CONSTRUCTORS = [ - lambda v: v, - lambda v: [v], - lambda v: [{'key': v}], -] - -class TestNamedTuple(unittest.TestCase): - def test_namedtuple_dumps(self): - for v in [Value(1), Point(1, 2), DuckValue(1), DuckPoint(1, 2)]: - d = v._asdict() - self.assertEqual(d, json.loads(json.dumps(v))) - self.assertEqual( - d, - json.loads(json.dumps(v, namedtuple_as_object=True))) - self.assertEqual(d, json.loads(json.dumps(v, tuple_as_array=False))) - self.assertEqual( - d, - json.loads(json.dumps(v, namedtuple_as_object=True, - tuple_as_array=False))) - - def test_namedtuple_dumps_false(self): - for v in [Value(1), Point(1, 2)]: - l = list(v) - self.assertEqual( - l, - json.loads(json.dumps(v, namedtuple_as_object=False))) - self.assertRaises(TypeError, json.dumps, v, - tuple_as_array=False, namedtuple_as_object=False) - - def test_namedtuple_dump(self): - for v in [Value(1), Point(1, 2), DuckValue(1), DuckPoint(1, 2)]: - d = v._asdict() - sio = StringIO() - json.dump(v, sio) - self.assertEqual(d, json.loads(sio.getvalue())) - sio = StringIO() - json.dump(v, sio, namedtuple_as_object=True) - self.assertEqual( - d, - json.loads(sio.getvalue())) - sio = StringIO() - json.dump(v, sio, tuple_as_array=False) - self.assertEqual(d, json.loads(sio.getvalue())) - sio = StringIO() - json.dump(v, sio, namedtuple_as_object=True, - tuple_as_array=False) - self.assertEqual( - d, - json.loads(sio.getvalue())) - - def test_namedtuple_dump_false(self): - for v in [Value(1), Point(1, 2)]: - l = list(v) - sio = StringIO() - json.dump(v, sio, namedtuple_as_object=False) - self.assertEqual( - l, - json.loads(sio.getvalue())) - self.assertRaises(TypeError, json.dump, v, StringIO(), - tuple_as_array=False, namedtuple_as_object=False) - - def test_asdict_not_callable_dump(self): - for f in CONSTRUCTORS: - self.assertRaises( - TypeError, - json.dump, - f(DeadDuck()), - StringIO(), - namedtuple_as_object=True - ) - sio = StringIO() - json.dump(f(DeadDict()), sio, namedtuple_as_object=True) - self.assertEqual( - json.dumps(f({})), - sio.getvalue()) - self.assertRaises( - TypeError, - json.dump, - f(Value), - StringIO(), - namedtuple_as_object=True - ) - - def test_asdict_not_callable_dumps(self): - for f in CONSTRUCTORS: - self.assertRaises(TypeError, - json.dumps, f(DeadDuck()), namedtuple_as_object=True) - self.assertRaises( - TypeError, - json.dumps, - f(Value), - namedtuple_as_object=True - ) - self.assertEqual( - json.dumps(f({})), - json.dumps(f(DeadDict()), namedtuple_as_object=True)) - - def test_asdict_unbound_method_dumps(self): - for f in CONSTRUCTORS: - self.assertEqual( - json.dumps(f(Value), default=lambda v: v.__name__), - json.dumps(f(Value.__name__)) - ) - - def test_asdict_does_not_return_dict(self): - if not mock: - if hasattr(unittest, "SkipTest"): - raise unittest.SkipTest("unittest.mock required") - else: - print("unittest.mock not available") - return - fake = mock.Mock() - self.assertTrue(hasattr(fake, '_asdict')) - self.assertTrue(callable(fake._asdict)) - self.assertFalse(isinstance(fake._asdict(), dict)) - # https://github.com/simplejson/simplejson/pull/284 - # when running under a debug build of CPython (COPTS=-UNDEBUG) - # a C assertion could fire due to an unchecked error of an PyDict - # API call on a non-dict internally in _speedups.c. Without a debug - # build of CPython this test likely passes either way despite the - # potential for internal data corruption. Getting it to crash in - # a debug build is not always easy either as it requires an - # assert(!PyErr_Occurred()) that could fire later on. - with self.assertRaises(TypeError): - json.dumps({23: fake}, namedtuple_as_object=True, for_json=False) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_pass1.py b/contrib/python/simplejson/py2/simplejson/tests/test_pass1.py deleted file mode 100644 index f0b5b10e761..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_pass1.py +++ /dev/null @@ -1,71 +0,0 @@ -from unittest import TestCase - -import simplejson as json - -# from http://json.org/JSON_checker/test/pass1.json -JSON = r''' -[ - "JSON Test Pattern pass1", - {"object with 1 member":["array with 1 element"]}, - {}, - [], - -42, - true, - false, - null, - { - "integer": 1234567890, - "real": -9876.543210, - "e": 0.123456789e-12, - "E": 1.234567890E+34, - "": 23456789012E66, - "zero": 0, - "one": 1, - "space": " ", - "quote": "\"", - "backslash": "\\", - "controls": "\b\f\n\r\t", - "slash": "/ & \/", - "alpha": "abcdefghijklmnopqrstuvwyz", - "ALPHA": "ABCDEFGHIJKLMNOPQRSTUVWYZ", - "digit": "0123456789", - "special": "`1~!@#$%^&*()_+-={':[,]}|;.</>?", - "hex": "\u0123\u4567\u89AB\uCDEF\uabcd\uef4A", - "true": true, - "false": false, - "null": null, - "array":[ ], - "object":{ }, - "address": "50 St. James Street", - "url": "http://www.JSON.org/", - "comment": "// /* <!-- --", - "# -- --> */": " ", - " s p a c e d " :[1,2 , 3 - -, - -4 , 5 , 6 ,7 ],"compact": [1,2,3,4,5,6,7], - "jsontext": "{\"object with 1 member\":[\"array with 1 element\"]}", - "quotes": "" \u0022 %22 0x22 034 "", - "\/\\\"\uCAFE\uBABE\uAB98\uFCDE\ubcda\uef4A\b\f\n\r\t`1~!@#$%^&*()_+-=[]{}|;:',./<>?" -: "A key can be any string" - }, - 0.5 ,98.6 -, -99.44 -, - -1066, -1e1, -0.1e1, -1e-1, -1e00,2e+00,2e-00 -,"rosebud"] -''' - -class TestPass1(TestCase): - def test_parse(self): - # test in/out equivalence and parsing - res = json.loads(JSON) - out = json.dumps(res) - self.assertEqual(res, json.loads(out)) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_pass2.py b/contrib/python/simplejson/py2/simplejson/tests/test_pass2.py deleted file mode 100644 index 5d812b3bbe2..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_pass2.py +++ /dev/null @@ -1,14 +0,0 @@ -from unittest import TestCase -import simplejson as json - -# from http://json.org/JSON_checker/test/pass2.json -JSON = r''' -[[[[[[[[[[[[[[[[[[["Not too deep"]]]]]]]]]]]]]]]]]]] -''' - -class TestPass2(TestCase): - def test_parse(self): - # test in/out equivalence and parsing - res = json.loads(JSON) - out = json.dumps(res) - self.assertEqual(res, json.loads(out)) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_pass3.py b/contrib/python/simplejson/py2/simplejson/tests/test_pass3.py deleted file mode 100644 index 821d60b22cc..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_pass3.py +++ /dev/null @@ -1,20 +0,0 @@ -from unittest import TestCase - -import simplejson as json - -# from http://json.org/JSON_checker/test/pass3.json -JSON = r''' -{ - "JSON Test Pattern pass3": { - "The outermost value": "must be an object or array.", - "In this test": "It is an object." - } -} -''' - -class TestPass3(TestCase): - def test_parse(self): - # test in/out equivalence and parsing - res = json.loads(JSON) - out = json.dumps(res) - self.assertEqual(res, json.loads(out)) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_raw_json.py b/contrib/python/simplejson/py2/simplejson/tests/test_raw_json.py deleted file mode 100644 index 1dfcc2c5f95..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_raw_json.py +++ /dev/null @@ -1,47 +0,0 @@ -import unittest -import simplejson as json - -dct1 = { - 'key1': 'value1' -} - -dct2 = { - 'key2': 'value2', - 'd1': dct1 -} - -dct3 = { - 'key2': 'value2', - 'd1': json.dumps(dct1) -} - -dct4 = { - 'key2': 'value2', - 'd1': json.RawJSON(json.dumps(dct1)) -} - - -class TestRawJson(unittest.TestCase): - - def test_normal_str(self): - self.assertNotEqual(json.dumps(dct2), json.dumps(dct3)) - - def test_raw_json_str(self): - self.assertEqual(json.dumps(dct2), json.dumps(dct4)) - self.assertEqual(dct2, json.loads(json.dumps(dct4))) - - def test_list(self): - self.assertEqual( - json.dumps([dct2]), - json.dumps([json.RawJSON(json.dumps(dct2))])) - self.assertEqual( - [dct2], - json.loads(json.dumps([json.RawJSON(json.dumps(dct2))]))) - - def test_direct(self): - self.assertEqual( - json.dumps(dct2), - json.dumps(json.RawJSON(json.dumps(dct2)))) - self.assertEqual( - dct2, - json.loads(json.dumps(json.RawJSON(json.dumps(dct2))))) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_recursion.py b/contrib/python/simplejson/py2/simplejson/tests/test_recursion.py deleted file mode 100644 index 662eb667ece..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_recursion.py +++ /dev/null @@ -1,67 +0,0 @@ -from unittest import TestCase - -import simplejson as json - -class JSONTestObject: - pass - - -class RecursiveJSONEncoder(json.JSONEncoder): - recurse = False - def default(self, o): - if o is JSONTestObject: - if self.recurse: - return [JSONTestObject] - else: - return 'JSONTestObject' - return json.JSONEncoder.default(o) - - -class TestRecursion(TestCase): - def test_listrecursion(self): - x = [] - x.append(x) - try: - json.dumps(x) - except ValueError: - pass - else: - self.fail("didn't raise ValueError on list recursion") - x = [] - y = [x] - x.append(y) - try: - json.dumps(x) - except ValueError: - pass - else: - self.fail("didn't raise ValueError on alternating list recursion") - y = [] - x = [y, y] - # ensure that the marker is cleared - json.dumps(x) - - def test_dictrecursion(self): - x = {} - x["test"] = x - try: - json.dumps(x) - except ValueError: - pass - else: - self.fail("didn't raise ValueError on dict recursion") - x = {} - y = {"a": x, "b": x} - # ensure that the marker is cleared - json.dumps(y) - - def test_defaultrecursion(self): - enc = RecursiveJSONEncoder() - self.assertEqual(enc.encode(JSONTestObject), '"JSONTestObject"') - enc.recurse = True - try: - enc.encode(JSONTestObject) - except ValueError: - pass - else: - self.fail("didn't raise ValueError on default recursion") diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_scanstring.py b/contrib/python/simplejson/py2/simplejson/tests/test_scanstring.py deleted file mode 100644 index 1f544834209..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_scanstring.py +++ /dev/null @@ -1,200 +0,0 @@ -import sys -from unittest import TestCase - -import simplejson as json -import simplejson.decoder -from simplejson.compat import b, PY3 - -class TestScanString(TestCase): - # The bytes type is intentionally not used in most of these tests - # under Python 3 because the decoder immediately coerces to str before - # calling scanstring. In Python 2 we are testing the code paths - # for both unicode and str. - # - # The reason this is done is because Python 3 would require - # entirely different code paths for parsing bytes and str. - # - def test_py_scanstring(self): - self._test_scanstring(simplejson.decoder.py_scanstring) - - def test_c_scanstring(self): - if not simplejson.decoder.c_scanstring: - return - self._test_scanstring(simplejson.decoder.c_scanstring) - - self.assertTrue(isinstance(simplejson.decoder.c_scanstring('""', 0)[0], str)) - - def _test_scanstring(self, scanstring): - if sys.maxunicode == 65535: - self.assertEqual( - scanstring(u'"z\U0001d120x"', 1, None, True), - (u'z\U0001d120x', 6)) - else: - self.assertEqual( - scanstring(u'"z\U0001d120x"', 1, None, True), - (u'z\U0001d120x', 5)) - - self.assertEqual( - scanstring('"\\u007b"', 1, None, True), - (u'{', 8)) - - self.assertEqual( - scanstring('"A JSON payload should be an object or array, not a string."', 1, None, True), - (u'A JSON payload should be an object or array, not a string.', 60)) - - self.assertEqual( - scanstring('["Unclosed array"', 2, None, True), - (u'Unclosed array', 17)) - - self.assertEqual( - scanstring('["extra comma",]', 2, None, True), - (u'extra comma', 14)) - - self.assertEqual( - scanstring('["double extra comma",,]', 2, None, True), - (u'double extra comma', 21)) - - self.assertEqual( - scanstring('["Comma after the close"],', 2, None, True), - (u'Comma after the close', 24)) - - self.assertEqual( - scanstring('["Extra close"]]', 2, None, True), - (u'Extra close', 14)) - - self.assertEqual( - scanstring('{"Extra comma": true,}', 2, None, True), - (u'Extra comma', 14)) - - self.assertEqual( - scanstring('{"Extra value after close": true} "misplaced quoted value"', 2, None, True), - (u'Extra value after close', 26)) - - self.assertEqual( - scanstring('{"Illegal expression": 1 + 2}', 2, None, True), - (u'Illegal expression', 21)) - - self.assertEqual( - scanstring('{"Illegal invocation": alert()}', 2, None, True), - (u'Illegal invocation', 21)) - - self.assertEqual( - scanstring('{"Numbers cannot have leading zeroes": 013}', 2, None, True), - (u'Numbers cannot have leading zeroes', 37)) - - self.assertEqual( - scanstring('{"Numbers cannot be hex": 0x14}', 2, None, True), - (u'Numbers cannot be hex', 24)) - - self.assertEqual( - scanstring('[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]', 21, None, True), - (u'Too deep', 30)) - - self.assertEqual( - scanstring('{"Missing colon" null}', 2, None, True), - (u'Missing colon', 16)) - - self.assertEqual( - scanstring('{"Double colon":: null}', 2, None, True), - (u'Double colon', 15)) - - self.assertEqual( - scanstring('{"Comma instead of colon", null}', 2, None, True), - (u'Comma instead of colon', 25)) - - self.assertEqual( - scanstring('["Colon instead of comma": false]', 2, None, True), - (u'Colon instead of comma', 25)) - - self.assertEqual( - scanstring('["Bad value", truth]', 2, None, True), - (u'Bad value', 12)) - - for c in map(chr, range(0x00, 0x1f)): - self.assertEqual( - scanstring(c + '"', 0, None, False), - (c, 2)) - self.assertRaises( - ValueError, - scanstring, c + '"', 0, None, True) - - self.assertRaises(ValueError, scanstring, '', 0, None, True) - self.assertRaises(ValueError, scanstring, 'a', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\u', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\u0', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\u01', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\u012', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\u0123', 0, None, True) - if sys.maxunicode > 65535: - self.assertRaises(ValueError, - scanstring, '\\ud834\\u"', 0, None, True) - self.assertRaises(ValueError, - scanstring, '\\ud834\\x0123"', 0, None, True) - - self.assertRaises(json.JSONDecodeError, scanstring, '\\u-123"', 0, None, True) - # SJ-PT-23-01: Invalid Handling of Broken Unicode Escape Sequences - self.assertRaises(json.JSONDecodeError, scanstring, '\\u EDD"', 0, None, True) - - def test_issue3623(self): - self.assertRaises(ValueError, json.decoder.scanstring, "xxx", 1, - "xxx") - self.assertRaises(UnicodeDecodeError, - json.encoder.encode_basestring_ascii, b("xx\xff")) - - def test_overflow(self): - # Python 2.5 does not have maxsize, Python 3 does not have maxint - maxsize = getattr(sys, 'maxsize', getattr(sys, 'maxint', None)) - assert maxsize is not None - self.assertRaises(OverflowError, json.decoder.scanstring, "xxx", - maxsize + 1) - - def test_surrogates(self): - scanstring = json.decoder.scanstring - - def assertScan(given, expect, test_utf8=True): - givens = [given] - if not PY3 and test_utf8: - givens.append(given.encode('utf8')) - for given in givens: - (res, count) = scanstring(given, 1, None, True) - self.assertEqual(len(given), count) - self.assertEqual(res, expect) - - assertScan( - u'"z\\ud834\\u0079x"', - u'z\ud834yx') - assertScan( - u'"z\\ud834\\udd20x"', - u'z\U0001d120x') - assertScan( - u'"z\\ud834\\ud834\\udd20x"', - u'z\ud834\U0001d120x') - assertScan( - u'"z\\ud834x"', - u'z\ud834x') - assertScan( - u'"z\\udd20x"', - u'z\udd20x') - assertScan( - u'"z\ud834x"', - u'z\ud834x') - # It may look strange to join strings together, but Python is drunk. - # https://gist.github.com/etrepum/5538443 - assertScan( - u'"z\\ud834\udd20x12345"', - u''.join([u'z\ud834', u'\udd20x12345'])) - assertScan( - u'"z\ud834\\udd20x"', - u''.join([u'z\ud834', u'\udd20x'])) - # these have different behavior given UTF8 input, because the surrogate - # pair may be joined (in maxunicode > 65535 builds) - assertScan( - u''.join([u'"z\ud834', u'\udd20x"']), - u''.join([u'z\ud834', u'\udd20x']), - test_utf8=False) - - self.assertRaises(ValueError, - scanstring, u'"z\\ud83x"', 1, None, True) - self.assertRaises(ValueError, - scanstring, u'"z\\ud834\\udd2x"', 1, None, True) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_separators.py b/contrib/python/simplejson/py2/simplejson/tests/test_separators.py deleted file mode 100644 index 91b4d4fb6f3..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_separators.py +++ /dev/null @@ -1,42 +0,0 @@ -import textwrap -from unittest import TestCase - -import simplejson as json - - -class TestSeparators(TestCase): - def test_separators(self): - h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', 'i-vhbjkhnth', - {'nifty': 87}, {'field': 'yes', 'morefield': False} ] - - expect = textwrap.dedent("""\ - [ - [ - "blorpie" - ] , - [ - "whoops" - ] , - [] , - "d-shtaeou" , - "d-nthiouh" , - "i-vhbjkhnth" , - { - "nifty" : 87 - } , - { - "field" : "yes" , - "morefield" : false - } - ]""") - - - d1 = json.dumps(h) - d2 = json.dumps(h, indent=' ', sort_keys=True, separators=(' ,', ' : ')) - - h1 = json.loads(d1) - h2 = json.loads(d2) - - self.assertEqual(h1, h) - self.assertEqual(h2, h) - self.assertEqual(d2, expect) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_speedups.py b/contrib/python/simplejson/py2/simplejson/tests/test_speedups.py deleted file mode 100644 index 8b146df9a05..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_speedups.py +++ /dev/null @@ -1,114 +0,0 @@ -from __future__ import with_statement - -import sys -import unittest -from unittest import TestCase - -import simplejson -from simplejson import encoder, decoder, scanner -from simplejson.compat import PY3, long_type, b - - -def has_speedups(): - return encoder.c_make_encoder is not None - - -def skip_if_speedups_missing(func): - def wrapper(*args, **kwargs): - if not has_speedups(): - if hasattr(unittest, 'SkipTest'): - raise unittest.SkipTest("C Extension not available") - else: - sys.stdout.write("C Extension not available") - return - return func(*args, **kwargs) - - return wrapper - - -class BadBool: - def __bool__(self): - 1/0 - __nonzero__ = __bool__ - - -class TestDecode(TestCase): - @skip_if_speedups_missing - def test_make_scanner(self): - self.assertRaises(AttributeError, scanner.c_make_scanner, 1) - - @skip_if_speedups_missing - def test_bad_bool_args(self): - def test(value): - decoder.JSONDecoder(strict=BadBool()).decode(value) - self.assertRaises(ZeroDivisionError, test, '""') - self.assertRaises(ZeroDivisionError, test, '{}') - if not PY3: - self.assertRaises(ZeroDivisionError, test, u'""') - self.assertRaises(ZeroDivisionError, test, u'{}') - -class TestEncode(TestCase): - @skip_if_speedups_missing - def test_make_encoder(self): - self.assertRaises( - TypeError, - encoder.c_make_encoder, - None, - ("\xCD\x7D\x3D\x4E\x12\x4C\xF9\x79\xD7" - "\x52\xBA\x82\xF2\x27\x4A\x7D\xA0\xCA\x75"), - None - ) - - @skip_if_speedups_missing - def test_bad_str_encoder(self): - # Issue #31505: There shouldn't be an assertion failure in case - # c_make_encoder() receives a bad encoder() argument. - import decimal - def bad_encoder1(*args): - return None - enc = encoder.c_make_encoder( - None, lambda obj: str(obj), - bad_encoder1, None, ': ', ', ', - False, False, False, {}, False, False, False, - None, None, 'utf-8', False, False, decimal.Decimal, False) - self.assertRaises(TypeError, enc, 'spam', 4) - self.assertRaises(TypeError, enc, {'spam': 42}, 4) - - def bad_encoder2(*args): - 1/0 - enc = encoder.c_make_encoder( - None, lambda obj: str(obj), - bad_encoder2, None, ': ', ', ', - False, False, False, {}, False, False, False, - None, None, 'utf-8', False, False, decimal.Decimal, False) - self.assertRaises(ZeroDivisionError, enc, 'spam', 4) - - @skip_if_speedups_missing - def test_bad_bool_args(self): - def test(name): - encoder.JSONEncoder(**{name: BadBool()}).encode({}) - self.assertRaises(ZeroDivisionError, test, 'skipkeys') - self.assertRaises(ZeroDivisionError, test, 'ensure_ascii') - self.assertRaises(ZeroDivisionError, test, 'check_circular') - self.assertRaises(ZeroDivisionError, test, 'allow_nan') - self.assertRaises(ZeroDivisionError, test, 'sort_keys') - self.assertRaises(ZeroDivisionError, test, 'use_decimal') - self.assertRaises(ZeroDivisionError, test, 'namedtuple_as_object') - self.assertRaises(ZeroDivisionError, test, 'tuple_as_array') - self.assertRaises(ZeroDivisionError, test, 'bigint_as_string') - self.assertRaises(ZeroDivisionError, test, 'for_json') - self.assertRaises(ZeroDivisionError, test, 'ignore_nan') - self.assertRaises(ZeroDivisionError, test, 'iterable_as_array') - - @skip_if_speedups_missing - def test_int_as_string_bitcount_overflow(self): - long_count = long_type(2)**32+31 - def test(): - encoder.JSONEncoder(int_as_string_bitcount=long_count).encode(0) - self.assertRaises((TypeError, OverflowError), test) - - if PY3: - @skip_if_speedups_missing - def test_bad_encoding(self): - with self.assertRaises(UnicodeEncodeError): - encoder.JSONEncoder(encoding='\udcff').encode({b('key'): 123}) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_str_subclass.py b/contrib/python/simplejson/py2/simplejson/tests/test_str_subclass.py deleted file mode 100644 index b6c8351f264..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_str_subclass.py +++ /dev/null @@ -1,21 +0,0 @@ -from unittest import TestCase - -import simplejson -from simplejson.compat import text_type - -# Tests for issue demonstrated in https://github.com/simplejson/simplejson/issues/144 -class WonkyTextSubclass(text_type): - def __getslice__(self, start, end): - return self.__class__('not what you wanted!') - -class TestStrSubclass(TestCase): - def test_dump_load(self): - for s in ['', '"hello"', 'text', u'\u005c']: - self.assertEqual( - s, - simplejson.loads(simplejson.dumps(WonkyTextSubclass(s)))) - - self.assertEqual( - s, - simplejson.loads(simplejson.dumps(WonkyTextSubclass(s), - ensure_ascii=False))) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_subclass.py b/contrib/python/simplejson/py2/simplejson/tests/test_subclass.py deleted file mode 100644 index 2bae3b6309e..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_subclass.py +++ /dev/null @@ -1,37 +0,0 @@ -from unittest import TestCase -import simplejson as json - -from decimal import Decimal - -class AlternateInt(int): - def __repr__(self): - return 'invalid json' - __str__ = __repr__ - - -class AlternateFloat(float): - def __repr__(self): - return 'invalid json' - __str__ = __repr__ - - -# class AlternateDecimal(Decimal): -# def __repr__(self): -# return 'invalid json' - - -class TestSubclass(TestCase): - def test_int(self): - self.assertEqual(json.dumps(AlternateInt(1)), '1') - self.assertEqual(json.dumps(AlternateInt(-1)), '-1') - self.assertEqual(json.loads(json.dumps({AlternateInt(1): 1})), {'1': 1}) - - def test_float(self): - self.assertEqual(json.dumps(AlternateFloat(1.0)), '1.0') - self.assertEqual(json.dumps(AlternateFloat(-1.0)), '-1.0') - self.assertEqual(json.loads(json.dumps({AlternateFloat(1.0): 1})), {'1.0': 1}) - - # NOTE: Decimal subclasses are not supported as-is - # def test_decimal(self): - # self.assertEqual(json.dumps(AlternateDecimal('1.0')), '1.0') - # self.assertEqual(json.dumps(AlternateDecimal('-1.0')), '-1.0') diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_tool.py b/contrib/python/simplejson/py2/simplejson/tests/test_tool.py deleted file mode 100644 index 914bff8bca5..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_tool.py +++ /dev/null @@ -1,114 +0,0 @@ -from __future__ import with_statement -import os -import sys -import textwrap -import unittest -import subprocess -import tempfile -try: - # Python 3.x - from test.support import strip_python_stderr -except ImportError: - # Python 2.6+ - try: - from test.test_support import strip_python_stderr - except ImportError: - # Python 2.5 - import re - def strip_python_stderr(stderr): - return re.sub( - r"\[\d+ refs\]\r?\n?$".encode(), - "".encode(), - stderr).strip() - -def open_temp_file(): - if sys.version_info >= (2, 6): - file = tempfile.NamedTemporaryFile(delete=False) - filename = file.name - else: - fd, filename = tempfile.mkstemp() - file = os.fdopen(fd, 'w+b') - return file, filename - -class TestTool(unittest.TestCase): - data = """ - - [["blorpie"],[ "whoops" ] , [ - ],\t"d-shtaeou",\r"d-nthiouh", - "i-vhbjkhnth", {"nifty":87}, {"morefield" :\tfalse,"field" - :"yes"} ] - """ - - expect = textwrap.dedent("""\ - [ - [ - "blorpie" - ], - [ - "whoops" - ], - [], - "d-shtaeou", - "d-nthiouh", - "i-vhbjkhnth", - { - "nifty": 87 - }, - { - "field": "yes", - "morefield": false - } - ] - """) - - def runTool(self, args=None, data=None): - argv = [sys.executable, '-m', 'simplejson.tool'] - if args: - argv.extend(args) - proc = subprocess.Popen(argv, - stdin=subprocess.PIPE, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE) - out, err = proc.communicate(data) - self.assertEqual(strip_python_stderr(err), ''.encode()) - self.assertEqual(proc.returncode, 0) - return out.decode('utf8').splitlines() - - def test_stdin_stdout(self): - self.assertEqual( - self.runTool(data=self.data.encode()), - self.expect.splitlines()) - - def test_infile_stdout(self): - infile, infile_name = open_temp_file() - try: - infile.write(self.data.encode()) - infile.close() - self.assertEqual( - self.runTool(args=[infile_name]), - self.expect.splitlines()) - finally: - os.unlink(infile_name) - - def test_infile_outfile(self): - infile, infile_name = open_temp_file() - try: - infile.write(self.data.encode()) - infile.close() - # outfile will get overwritten by tool, so the delete - # may not work on some platforms. Do it manually. - outfile, outfile_name = open_temp_file() - try: - outfile.close() - self.assertEqual( - self.runTool(args=[infile_name, outfile_name]), - []) - with open(outfile_name, 'rb') as f: - self.assertEqual( - f.read().decode('utf8').splitlines(), - self.expect.splitlines() - ) - finally: - os.unlink(outfile_name) - finally: - os.unlink(infile_name) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_tuple.py b/contrib/python/simplejson/py2/simplejson/tests/test_tuple.py deleted file mode 100644 index 4ad7b0e861b..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_tuple.py +++ /dev/null @@ -1,47 +0,0 @@ -import unittest - -from simplejson.compat import StringIO -import simplejson as json - -class TestTuples(unittest.TestCase): - def test_tuple_array_dumps(self): - t = (1, 2, 3) - expect = json.dumps(list(t)) - # Default is True - self.assertEqual(expect, json.dumps(t)) - self.assertEqual(expect, json.dumps(t, tuple_as_array=True)) - self.assertRaises(TypeError, json.dumps, t, tuple_as_array=False) - # Ensure that the "default" does not get called - self.assertEqual(expect, json.dumps(t, default=repr)) - self.assertEqual(expect, json.dumps(t, tuple_as_array=True, - default=repr)) - # Ensure that the "default" gets called - self.assertEqual( - json.dumps(repr(t)), - json.dumps(t, tuple_as_array=False, default=repr)) - - def test_tuple_array_dump(self): - t = (1, 2, 3) - expect = json.dumps(list(t)) - # Default is True - sio = StringIO() - json.dump(t, sio) - self.assertEqual(expect, sio.getvalue()) - sio = StringIO() - json.dump(t, sio, tuple_as_array=True) - self.assertEqual(expect, sio.getvalue()) - self.assertRaises(TypeError, json.dump, t, StringIO(), - tuple_as_array=False) - # Ensure that the "default" does not get called - sio = StringIO() - json.dump(t, sio, default=repr) - self.assertEqual(expect, sio.getvalue()) - sio = StringIO() - json.dump(t, sio, tuple_as_array=True, default=repr) - self.assertEqual(expect, sio.getvalue()) - # Ensure that the "default" gets called - sio = StringIO() - json.dump(t, sio, tuple_as_array=False, default=repr) - self.assertEqual( - json.dumps(repr(t)), - sio.getvalue()) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_unicode.py b/contrib/python/simplejson/py2/simplejson/tests/test_unicode.py deleted file mode 100644 index 0c7b1a6c444..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tests/test_unicode.py +++ /dev/null @@ -1,154 +0,0 @@ -import sys -import codecs -from unittest import TestCase - -import simplejson as json -from simplejson.compat import unichr, text_type, b, BytesIO - -class TestUnicode(TestCase): - def test_encoding1(self): - encoder = json.JSONEncoder(encoding='utf-8') - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - s = u.encode('utf-8') - ju = encoder.encode(u) - js = encoder.encode(s) - self.assertEqual(ju, js) - - def test_encoding2(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - s = u.encode('utf-8') - ju = json.dumps(u, encoding='utf-8') - js = json.dumps(s, encoding='utf-8') - self.assertEqual(ju, js) - - def test_encoding3(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps(u) - self.assertEqual(j, '"\\u03b1\\u03a9"') - - def test_encoding4(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps([u]) - self.assertEqual(j, '["\\u03b1\\u03a9"]') - - def test_encoding5(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps(u, ensure_ascii=False) - self.assertEqual(j, u'"' + u + u'"') - - def test_encoding6(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps([u], ensure_ascii=False) - self.assertEqual(j, u'["' + u + u'"]') - - def test_big_unicode_encode(self): - u = u'\U0001d120' - self.assertEqual(json.dumps(u), '"\\ud834\\udd20"') - self.assertEqual(json.dumps(u, ensure_ascii=False), u'"\U0001d120"') - - def test_big_unicode_decode(self): - u = u'z\U0001d120x' - self.assertEqual(json.loads('"' + u + '"'), u) - self.assertEqual(json.loads('"z\\ud834\\udd20x"'), u) - - def test_unicode_decode(self): - for i in range(0, 0xd7ff): - u = unichr(i) - #s = '"\\u{0:04x}"'.format(i) - s = '"\\u%04x"' % (i,) - self.assertEqual(json.loads(s), u) - - def test_object_pairs_hook_with_unicode(self): - s = u'{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}' - p = [(u"xkd", 1), (u"kcw", 2), (u"art", 3), (u"hxm", 4), - (u"qrt", 5), (u"pad", 6), (u"hoy", 7)] - self.assertEqual(json.loads(s), eval(s)) - self.assertEqual(json.loads(s, object_pairs_hook=lambda x: x), p) - od = json.loads(s, object_pairs_hook=json.OrderedDict) - self.assertEqual(od, json.OrderedDict(p)) - self.assertEqual(type(od), json.OrderedDict) - # the object_pairs_hook takes priority over the object_hook - self.assertEqual(json.loads(s, - object_pairs_hook=json.OrderedDict, - object_hook=lambda x: None), - json.OrderedDict(p)) - - - def test_default_encoding(self): - self.assertEqual(json.loads(u'{"a": "\xe9"}'.encode('utf-8')), - {'a': u'\xe9'}) - - def test_unicode_preservation(self): - self.assertEqual(type(json.loads(u'""')), text_type) - self.assertEqual(type(json.loads(u'"a"')), text_type) - self.assertEqual(type(json.loads(u'["a"]')[0]), text_type) - - def test_ensure_ascii_false_returns_unicode(self): - # http://code.google.com/p/simplejson/issues/detail?id=48 - self.assertEqual(type(json.dumps([], ensure_ascii=False)), text_type) - self.assertEqual(type(json.dumps(0, ensure_ascii=False)), text_type) - self.assertEqual(type(json.dumps({}, ensure_ascii=False)), text_type) - self.assertEqual(type(json.dumps("", ensure_ascii=False)), text_type) - - def test_ensure_ascii_false_bytestring_encoding(self): - # http://code.google.com/p/simplejson/issues/detail?id=48 - doc1 = {u'quux': b('Arr\xc3\xaat sur images')} - doc2 = {u'quux': u'Arr\xeat sur images'} - doc_ascii = '{"quux": "Arr\\u00eat sur images"}' - doc_unicode = u'{"quux": "Arr\xeat sur images"}' - self.assertEqual(json.dumps(doc1), doc_ascii) - self.assertEqual(json.dumps(doc2), doc_ascii) - self.assertEqual(json.dumps(doc1, ensure_ascii=False), doc_unicode) - self.assertEqual(json.dumps(doc2, ensure_ascii=False), doc_unicode) - - def test_ensure_ascii_linebreak_encoding(self): - # http://timelessrepo.com/json-isnt-a-javascript-subset - s1 = u'\u2029\u2028' - s2 = s1.encode('utf8') - expect = '"\\u2029\\u2028"' - expect_non_ascii = u'"\u2029\u2028"' - self.assertEqual(json.dumps(s1), expect) - self.assertEqual(json.dumps(s2), expect) - self.assertEqual(json.dumps(s1, ensure_ascii=False), expect_non_ascii) - self.assertEqual(json.dumps(s2, ensure_ascii=False), expect_non_ascii) - - def test_invalid_escape_sequences(self): - # incomplete escape sequence - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u1') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u12') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u123') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u1234') - # invalid escape sequence - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u123x"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u12x4"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u1x34"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ux234"') - if sys.maxunicode > 65535: - # invalid escape sequence for low surrogate - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u0"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u00"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u000"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u000x"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u00x0"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u0x00"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\ux000"') - - def test_ensure_ascii_still_works(self): - # in the ascii range, ensure that everything is the same - for c in map(unichr, range(0, 127)): - self.assertEqual( - json.dumps(c, ensure_ascii=False), - json.dumps(c)) - snowman = u'\N{SNOWMAN}' - self.assertEqual( - json.dumps(c, ensure_ascii=False), - '"' + c + '"') - - def test_strip_bom(self): - content = u"\u3053\u3093\u306b\u3061\u308f" - json_doc = codecs.BOM_UTF8 + b(json.dumps(content)) - self.assertEqual(json.load(BytesIO(json_doc)), content) - for doc in json_doc, json_doc.decode('utf8'): - self.assertEqual(json.loads(doc), content) diff --git a/contrib/python/simplejson/py2/simplejson/tool.py b/contrib/python/simplejson/py2/simplejson/tool.py deleted file mode 100644 index 062e8e2c181..00000000000 --- a/contrib/python/simplejson/py2/simplejson/tool.py +++ /dev/null @@ -1,42 +0,0 @@ -r"""Command-line tool to validate and pretty-print JSON - -Usage:: - - $ echo '{"json":"obj"}' | python -m simplejson.tool - { - "json": "obj" - } - $ echo '{ 1.2:3.4}' | python -m simplejson.tool - Expecting property name: line 1 column 2 (char 2) - -""" -from __future__ import with_statement -import sys -import simplejson as json - -def main(): - if len(sys.argv) == 1: - infile = sys.stdin - outfile = sys.stdout - elif len(sys.argv) == 2: - infile = open(sys.argv[1], 'r') - outfile = sys.stdout - elif len(sys.argv) == 3: - infile = open(sys.argv[1], 'r') - outfile = open(sys.argv[2], 'w') - else: - raise SystemExit(sys.argv[0] + " [infile [outfile]]") - with infile: - try: - obj = json.load(infile, - object_pairs_hook=json.OrderedDict, - use_decimal=True) - except ValueError: - raise SystemExit(sys.exc_info()[1]) - with outfile: - json.dump(obj, outfile, sort_keys=True, indent=' ', use_decimal=True) - outfile.write('\n') - - -if __name__ == '__main__': - main() diff --git a/contrib/python/simplejson/py2/tests/ya.make b/contrib/python/simplejson/py2/tests/ya.make deleted file mode 100644 index 23e25312213..00000000000 --- a/contrib/python/simplejson/py2/tests/ya.make +++ /dev/null @@ -1,47 +0,0 @@ -PY2TEST() - -PEERDIR( - contrib/python/simplejson -) - -SRCDIR( - contrib/python/simplejson/py2/simplejson/tests -) - -TEST_SRCS( - __init__.py - test_bigint_as_string.py - test_bitsize_int_as_string.py - test_check_circular.py - test_decimal.py - test_decode.py - test_default.py - test_dump.py - test_encode_basestring_ascii.py - test_encode_for_html.py - test_errors.py - test_fail.py - test_float.py - test_for_json.py - test_indent.py - test_item_sort_key.py - test_iterable.py - test_namedtuple.py - test_pass1.py - test_pass2.py - test_pass3.py - test_raw_json.py - test_recursion.py - test_scanstring.py - test_separators.py - test_speedups.py - test_str_subclass.py - test_subclass.py - # test_tool.py - test_tuple.py - test_unicode.py -) - -NO_LINT() - -END() diff --git a/contrib/python/simplejson/py2/ya.make b/contrib/python/simplejson/py2/ya.make deleted file mode 100644 index 27c6bf9e27f..00000000000 --- a/contrib/python/simplejson/py2/ya.make +++ /dev/null @@ -1,48 +0,0 @@ -# Generated by devtools/yamaker (pypi). - -PY2_LIBRARY() - -VERSION(3.20.2) - -LICENSE(MIT) - -NO_COMPILER_WARNINGS() - -NO_LINT() - -NO_CHECK_IMPORTS( - simplejson.ordered_dict -) - -SRCS( - simplejson/_speedups.c -) - -PY_REGISTER( - simplejson._speedups -) - -PY_SRCS( - TOP_LEVEL - simplejson/__init__.py - simplejson/compat.py - simplejson/decoder.py - simplejson/encoder.py - simplejson/errors.py - simplejson/ordered_dict.py - simplejson/raw_json.py - simplejson/scanner.py - simplejson/tool.py -) - -RESOURCE_FILES( - PREFIX contrib/python/simplejson/py2/ - .dist-info/METADATA - .dist-info/top_level.txt -) - -END() - -RECURSE_FOR_TESTS( - tests -) diff --git a/contrib/python/simplejson/py3/.dist-info/METADATA b/contrib/python/simplejson/py3/.dist-info/METADATA deleted file mode 100644 index 31a275ebf8c..00000000000 --- a/contrib/python/simplejson/py3/.dist-info/METADATA +++ /dev/null @@ -1,80 +0,0 @@ -Metadata-Version: 2.4 -Name: simplejson -Version: 3.20.2 -Summary: Simple, fast, extensible JSON encoder/decoder for Python -Home-page: https://github.com/simplejson/simplejson -Author: Bob Ippolito -Author-email: [email protected] -License: MIT License -Platform: any -Classifier: Development Status :: 5 - Production/Stable -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: MIT License -Classifier: License :: OSI Approved :: Academic Free License (AFL) -Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 2 -Classifier: Programming Language :: Python :: 2.5 -Classifier: Programming Language :: Python :: 2.6 -Classifier: Programming Language :: Python :: 2.7 -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.3 -Classifier: Programming Language :: Python :: 3.4 -Classifier: Programming Language :: Python :: 3.5 -Classifier: Programming Language :: Python :: 3.6 -Classifier: Programming Language :: Python :: 3.7 -Classifier: Programming Language :: Python :: 3.8 -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Programming Language :: Python :: Implementation :: CPython -Classifier: Programming Language :: Python :: Implementation :: PyPy -Classifier: Topic :: Software Development :: Libraries :: Python Modules -Requires-Python: >=2.5, !=3.0.*, !=3.1.*, !=3.2.* -License-File: LICENSE.txt -Dynamic: author -Dynamic: author-email -Dynamic: classifier -Dynamic: description -Dynamic: home-page -Dynamic: license -Dynamic: license-file -Dynamic: platform -Dynamic: requires-python -Dynamic: summary - -simplejson ----------- - -simplejson is a simple, fast, complete, correct and extensible -JSON <http://json.org> encoder and decoder for Python 3.3+ -with legacy support for Python 2.5+. It is pure Python code -with no dependencies, but includes an optional C extension -for a serious speed boost. - -The latest documentation for simplejson can be read online here: -https://simplejson.readthedocs.io/ - -simplejson is the externally maintained development version of the -json library included with Python (since 2.6). This version is tested -with the latest Python 3.8 and maintains backwards compatibility -with Python 3.3+ and the legacy Python 2.5 - Python 2.7 releases. - -The encoder can be specialized to provide serialization in any kind of -situation, without any special support by the objects to be serialized -(somewhat like pickle). This is best done with the ``default`` kwarg -to dumps. - -The decoder can handle incoming JSON strings of any specified encoding -(UTF-8 by default). It can also be specialized to post-process JSON -objects with the ``object_hook`` or ``object_pairs_hook`` kwargs. This -is particularly useful for implementing protocols such as JSON-RPC -that have a richer type system than JSON itself. - -For those of you that have legacy systems to maintain, there is a -very old fork of simplejson in the `python2.2`_ branch that supports -Python 2.2. This is based on a very old version of simplejson, -is not maintained, and should only be used as a last resort. - -.. _python2.2: https://github.com/simplejson/simplejson/tree/python2.2 diff --git a/contrib/python/simplejson/py3/.dist-info/top_level.txt b/contrib/python/simplejson/py3/.dist-info/top_level.txt deleted file mode 100644 index 322630ee75f..00000000000 --- a/contrib/python/simplejson/py3/.dist-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -simplejson diff --git a/contrib/python/simplejson/py3/LICENSE.txt b/contrib/python/simplejson/py3/LICENSE.txt deleted file mode 100644 index e05f49c3fd0..00000000000 --- a/contrib/python/simplejson/py3/LICENSE.txt +++ /dev/null @@ -1,79 +0,0 @@ -simplejson is dual-licensed software. It is available under the terms -of the MIT license, or the Academic Free License version 2.1. The full -text of each license agreement is included below. This code is also -licensed to the Python Software Foundation (PSF) under a Contributor -Agreement. - -MIT License -=========== - -Copyright (c) 2006 Bob Ippolito - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -Academic Free License v. 2.1 -============================ - -Copyright (c) 2006 Bob Ippolito. All rights reserved. - -This Academic Free License (the "License") applies to any original work of authorship (the "Original Work") whose owner (the "Licensor") has placed the following notice immediately following the copyright notice for the Original Work: - -Licensed under the Academic Free License version 2.1 - -1) Grant of Copyright License. Licensor hereby grants You a world-wide, royalty-free, non-exclusive, perpetual, sublicenseable license to do the following: - -a) to reproduce the Original Work in copies; - -b) to prepare derivative works ("Derivative Works") based upon the Original Work; - -c) to distribute copies of the Original Work and Derivative Works to the public; - -d) to perform the Original Work publicly; and - -e) to display the Original Work publicly. - -2) Grant of Patent License. Licensor hereby grants You a world-wide, royalty-free, non-exclusive, perpetual, sublicenseable license, under patent claims owned or controlled by the Licensor that are embodied in the Original Work as furnished by the Licensor, to make, use, sell and offer for sale the Original Work and Derivative Works. - -3) Grant of Source Code License. The term "Source Code" means the preferred form of the Original Work for making modifications to it and all available documentation describing how to modify the Original Work. Licensor hereby agrees to provide a machine-readable copy of the Source Code of the Original Work along with each copy of the Original Work that Licensor distributes. Licensor reserves the right to satisfy this obligation by placing a machine-readable copy of the Source Code in an information repository reasonably calculated to permit inexpensive and convenient access by You for as long as Licensor continues to distribute the Original Work, and by publishing the address of that information repository in a notice immediately following the copyright notice that applies to the Original Work. - -4) Exclusions From License Grant. Neither the names of Licensor, nor the names of any contributors to the Original Work, nor any of their trademarks or service marks, may be used to endorse or promote products derived from this Original Work without express prior written permission of the Licensor. Nothing in this License shall be deemed to grant any rights to trademarks, copyrights, patents, trade secrets or any other intellectual property of Licensor except as expressly stated herein. No patent license is granted to make, use, sell or offer to sell embodiments of any patent claims other than the licensed claims defined in Section 2. No right is granted to the trademarks of Licensor even if such marks are included in the Original Work. Nothing in this License shall be interpreted to prohibit Licensor from licensing under different terms from this License any Original Work that Licensor otherwise would have a right to license. - -5) This section intentionally omitted. - -6) Attribution Rights. You must retain, in the Source Code of any Derivative Works that You create, all copyright, patent or trademark notices from the Source Code of the Original Work, as well as any notices of licensing and any descriptive text identified therein as an "Attribution Notice." You must cause the Source Code for any Derivative Works that You create to carry a prominent Attribution Notice reasonably calculated to inform recipients that You have modified the Original Work. - -7) Warranty of Provenance and Disclaimer of Warranty. Licensor warrants that the copyright in and to the Original Work and the patent rights granted herein by Licensor are owned by the Licensor or are sublicensed to You under the terms of this License with the permission of the contributor(s) of those copyrights and patent rights. Except as expressly stated in the immediately proceeding sentence, the Original Work is provided under this License on an "AS IS" BASIS and WITHOUT WARRANTY, either express or implied, including, without limitation, the warranties of NON-INFRINGEMENT, MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY OF THE ORIGINAL WORK IS WITH YOU. This DISCLAIMER OF WARRANTY constitutes an essential part of this License. No license to Original Work is granted hereunder except under this disclaimer. - -8) Limitation of Liability. Under no circumstances and under no legal theory, whether in tort (including negligence), contract, or otherwise, shall the Licensor be liable to any person for any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or the use of the Original Work including, without limitation, damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses. This limitation of liability shall not apply to liability for death or personal injury resulting from Licensor's negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. - -9) Acceptance and Termination. If You distribute copies of the Original Work or a Derivative Work, You must make a reasonable effort under the circumstances to obtain the express assent of recipients to the terms of this License. Nothing else but this License (or another written agreement between Licensor and You) grants You permission to create Derivative Works based upon the Original Work or to exercise any of the rights granted in Section 1 herein, and any attempt to do so except under the terms of this License (or another written agreement between Licensor and You) is expressly prohibited by U.S. copyright law, the equivalent laws of other countries, and by international treaty. Therefore, by exercising any of the rights granted to You in Section 1 herein, You indicate Your acceptance of this License and all of its terms and conditions. - -10) Termination for Patent Action. This License shall terminate automatically and You may no longer exercise any of the rights granted to You by this License as of the date You commence an action, including a cross-claim or counterclaim, against Licensor or any licensee alleging that the Original Work infringes a patent. This termination provision shall not apply for an action alleging patent infringement by combinations of the Original Work with other software or hardware. - -11) Jurisdiction, Venue and Governing Law. Any action or suit relating to this License may be brought only in the courts of a jurisdiction wherein the Licensor resides or in which Licensor conducts its primary business, and under the laws of that jurisdiction excluding its conflict-of-law provisions. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any use of the Original Work outside the scope of this License or after its termination shall be subject to the requirements and penalties of the U.S. Copyright Act, 17 U.S.C. § 101 et seq., the equivalent laws of other countries, and international treaty. This section shall survive the termination of this License. - -12) Attorneys Fees. In any action to enforce the terms of this License or seeking damages relating thereto, the prevailing party shall be entitled to recover its costs and expenses, including, without limitation, reasonable attorneys' fees and costs incurred in connection with such action, including any appeal of such action. This section shall survive the termination of this License. - -13) Miscellaneous. This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. - -14) Definition of "You" in This License. "You" throughout this License, whether in upper or lower case, means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License. For legal entities, "You" includes any entity that controls, is controlled by, or is under common control with you. For purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. - -15) Right to Use. You may use the Original Work in all ways not otherwise restricted or conditioned by this License or by law, and Licensor promises not to interfere with or be responsible for such uses by You. - -This license is Copyright (C) 2003-2004 Lawrence E. Rosen. All rights reserved. Permission is hereby granted to copy and distribute this license without modification. This license may not be modified without the express written permission of its copyright owner. diff --git a/contrib/python/simplejson/py3/README.rst b/contrib/python/simplejson/py3/README.rst deleted file mode 100644 index 6580ddac899..00000000000 --- a/contrib/python/simplejson/py3/README.rst +++ /dev/null @@ -1,34 +0,0 @@ -simplejson ----------- - -simplejson is a simple, fast, complete, correct and extensible -JSON <http://json.org> encoder and decoder for Python 3.3+ -with legacy support for Python 2.5+. It is pure Python code -with no dependencies, but includes an optional C extension -for a serious speed boost. - -The latest documentation for simplejson can be read online here: -https://simplejson.readthedocs.io/ - -simplejson is the externally maintained development version of the -json library included with Python (since 2.6). This version is tested -with the latest Python 3.8 and maintains backwards compatibility -with Python 3.3+ and the legacy Python 2.5 - Python 2.7 releases. - -The encoder can be specialized to provide serialization in any kind of -situation, without any special support by the objects to be serialized -(somewhat like pickle). This is best done with the ``default`` kwarg -to dumps. - -The decoder can handle incoming JSON strings of any specified encoding -(UTF-8 by default). It can also be specialized to post-process JSON -objects with the ``object_hook`` or ``object_pairs_hook`` kwargs. This -is particularly useful for implementing protocols such as JSON-RPC -that have a richer type system than JSON itself. - -For those of you that have legacy systems to maintain, there is a -very old fork of simplejson in the `python2.2`_ branch that supports -Python 2.2. This is based on a very old version of simplejson, -is not maintained, and should only be used as a last resort. - -.. _python2.2: https://github.com/simplejson/simplejson/tree/python2.2 diff --git a/contrib/python/simplejson/py3/simplejson/__init__.py b/contrib/python/simplejson/py3/simplejson/__init__.py deleted file mode 100644 index a526d8c7823..00000000000 --- a/contrib/python/simplejson/py3/simplejson/__init__.py +++ /dev/null @@ -1,562 +0,0 @@ -r"""JSON (JavaScript Object Notation) <http://json.org> is a subset of -JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data -interchange format. - -:mod:`simplejson` exposes an API familiar to users of the standard library -:mod:`marshal` and :mod:`pickle` modules. It is the externally maintained -version of the :mod:`json` library contained in Python 2.6, but maintains -compatibility back to Python 2.5 and (currently) has significant performance -advantages, even without using the optional C extension for speedups. - -Encoding basic Python object hierarchies:: - - >>> import simplejson as json - >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}]) - '["foo", {"bar": ["baz", null, 1.0, 2]}]' - >>> print(json.dumps("\"foo\bar")) - "\"foo\bar" - >>> print(json.dumps(u'\u1234')) - "\u1234" - >>> print(json.dumps('\\')) - "\\" - >>> print(json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True)) - {"a": 0, "b": 0, "c": 0} - >>> from simplejson.compat import StringIO - >>> io = StringIO() - >>> json.dump(['streaming API'], io) - >>> io.getvalue() - '["streaming API"]' - -Compact encoding:: - - >>> import simplejson as json - >>> obj = [1,2,3,{'4': 5, '6': 7}] - >>> json.dumps(obj, separators=(',',':'), sort_keys=True) - '[1,2,3,{"4":5,"6":7}]' - -Pretty printing:: - - >>> import simplejson as json - >>> print(json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=' ')) - { - "4": 5, - "6": 7 - } - -Decoding JSON:: - - >>> import simplejson as json - >>> obj = [u'foo', {u'bar': [u'baz', None, 1.0, 2]}] - >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj - True - >>> json.loads('"\\"foo\\bar"') == u'"foo\x08ar' - True - >>> from simplejson.compat import StringIO - >>> io = StringIO('["streaming API"]') - >>> json.load(io)[0] == 'streaming API' - True - -Specializing JSON object decoding:: - - >>> import simplejson as json - >>> def as_complex(dct): - ... if '__complex__' in dct: - ... return complex(dct['real'], dct['imag']) - ... return dct - ... - >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}', - ... object_hook=as_complex) - (1+2j) - >>> from decimal import Decimal - >>> json.loads('1.1', parse_float=Decimal) == Decimal('1.1') - True - -Specializing JSON object encoding:: - - >>> import simplejson as json - >>> def encode_complex(obj): - ... if isinstance(obj, complex): - ... return [obj.real, obj.imag] - ... raise TypeError('Object of type %s is not JSON serializable' % - ... obj.__class__.__name__) - ... - >>> json.dumps(2 + 1j, default=encode_complex) - '[2.0, 1.0]' - >>> json.JSONEncoder(default=encode_complex).encode(2 + 1j) - '[2.0, 1.0]' - >>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j)) - '[2.0, 1.0]' - -Using simplejson.tool from the shell to validate and pretty-print:: - - $ echo '{"json":"obj"}' | python -m simplejson.tool - { - "json": "obj" - } - $ echo '{ 1.2:3.4}' | python -m simplejson.tool - Expecting property name: line 1 column 3 (char 2) - -Parsing multiple documents serialized as JSON lines (newline-delimited JSON):: - - >>> import simplejson as json - >>> def loads_lines(docs): - ... for doc in docs.splitlines(): - ... yield json.loads(doc) - ... - >>> sum(doc["count"] for doc in loads_lines('{"count":1}\n{"count":2}\n{"count":3}\n')) - 6 - -Serializing multiple objects to JSON lines (newline-delimited JSON):: - - >>> import simplejson as json - >>> def dumps_lines(objs): - ... for obj in objs: - ... yield json.dumps(obj, separators=(',',':')) + '\n' - ... - >>> ''.join(dumps_lines([{'count': 1}, {'count': 2}, {'count': 3}])) - '{"count":1}\n{"count":2}\n{"count":3}\n' - -""" -from __future__ import absolute_import -__version__ = '3.20.2' -__all__ = [ - 'dump', 'dumps', 'load', 'loads', - 'JSONDecoder', 'JSONDecodeError', 'JSONEncoder', - 'OrderedDict', 'simple_first', 'RawJSON' -] - -__author__ = 'Bob Ippolito <[email protected]>' - -from decimal import Decimal - -from .errors import JSONDecodeError -from .raw_json import RawJSON -from .decoder import JSONDecoder -from .encoder import JSONEncoder, JSONEncoderForHTML -def _import_OrderedDict(): - import collections - try: - return collections.OrderedDict - except AttributeError: - from . import ordered_dict - return ordered_dict.OrderedDict -OrderedDict = _import_OrderedDict() - -def _import_c_make_encoder(): - try: - from ._speedups import make_encoder - return make_encoder - except ImportError: - return None - -_default_encoder = JSONEncoder() - -def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, - allow_nan=False, cls=None, indent=None, separators=None, - encoding='utf-8', default=None, use_decimal=True, - namedtuple_as_object=True, tuple_as_array=True, - bigint_as_string=False, sort_keys=False, item_sort_key=None, - for_json=False, ignore_nan=False, int_as_string_bitcount=None, - iterable_as_array=False, **kw): - """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a - ``.write()``-supporting file-like object). - - If *skipkeys* is true then ``dict`` keys that are not basic types - (``str``, ``int``, ``long``, ``float``, ``bool``, ``None``) - will be skipped instead of raising a ``TypeError``. - - If *ensure_ascii* is false (default: ``True``), then the output may - contain non-ASCII characters, so long as they do not need to be escaped - by JSON. When it is true, all non-ASCII characters are escaped. - - If *allow_nan* is true (default: ``False``), then out of range ``float`` - values (``nan``, ``inf``, ``-inf``) will be serialized to - their JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``) - instead of raising a ValueError. See - *ignore_nan* for ECMA-262 compliant behavior. - - If *indent* is a string, then JSON array elements and object members - will be pretty-printed with a newline followed by that string repeated - for each level of nesting. ``None`` (the default) selects the most compact - representation without any newlines. - - If specified, *separators* should be an - ``(item_separator, key_separator)`` tuple. The default is ``(', ', ': ')`` - if *indent* is ``None`` and ``(',', ': ')`` otherwise. To get the most - compact JSON representation, you should specify ``(',', ':')`` to eliminate - whitespace. - - *encoding* is the character encoding for str instances, default is UTF-8. - - *default(obj)* is a function that should return a serializable version - of obj or raise ``TypeError``. The default simply raises ``TypeError``. - - If *use_decimal* is true (default: ``True``) then decimal.Decimal - will be natively serialized to JSON with full precision. - - If *namedtuple_as_object* is true (default: ``True``), - :class:`tuple` subclasses with ``_asdict()`` methods will be encoded - as JSON objects. - - If *tuple_as_array* is true (default: ``True``), - :class:`tuple` (and subclasses) will be encoded as JSON arrays. - - If *iterable_as_array* is true (default: ``False``), - any object not in the above table that implements ``__iter__()`` - will be encoded as a JSON array. - - If *bigint_as_string* is true (default: ``False``), ints 2**53 and higher - or lower than -2**53 will be encoded as strings. This is to avoid the - rounding that happens in Javascript otherwise. Note that this is still a - lossy operation that will not round-trip correctly and should be used - sparingly. - - If *int_as_string_bitcount* is a positive number (n), then int of size - greater than or equal to 2**n or lower than or equal to -2**n will be - encoded as strings. - - If specified, *item_sort_key* is a callable used to sort the items in - each dictionary. This is useful if you want to sort items other than - in alphabetical order by key. This option takes precedence over - *sort_keys*. - - If *sort_keys* is true (default: ``False``), the output of dictionaries - will be sorted by item. - - If *for_json* is true (default: ``False``), objects with a ``for_json()`` - method will use the return value of that method for encoding as JSON - instead of the object. - - If *ignore_nan* is true (default: ``False``), then out of range - :class:`float` values (``nan``, ``inf``, ``-inf``) will be serialized as - ``null`` in compliance with the ECMA-262 specification. If true, this will - override *allow_nan*. - - To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the - ``.default()`` method to serialize additional types), specify it with - the ``cls`` kwarg. NOTE: You should use *default* or *for_json* instead - of subclassing whenever possible. - - """ - # cached encoder - if (not skipkeys and ensure_ascii and - check_circular and not allow_nan and - cls is None and indent is None and separators is None and - encoding == 'utf-8' and default is None and use_decimal - and namedtuple_as_object and tuple_as_array and not iterable_as_array - and not bigint_as_string and not sort_keys - and not item_sort_key and not for_json - and not ignore_nan and int_as_string_bitcount is None - and not kw - ): - iterable = _default_encoder.iterencode(obj) - else: - if cls is None: - cls = JSONEncoder - iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii, - check_circular=check_circular, allow_nan=allow_nan, indent=indent, - separators=separators, encoding=encoding, - default=default, use_decimal=use_decimal, - namedtuple_as_object=namedtuple_as_object, - tuple_as_array=tuple_as_array, - iterable_as_array=iterable_as_array, - bigint_as_string=bigint_as_string, - sort_keys=sort_keys, - item_sort_key=item_sort_key, - for_json=for_json, - ignore_nan=ignore_nan, - int_as_string_bitcount=int_as_string_bitcount, - **kw).iterencode(obj) - # could accelerate with writelines in some versions of Python, at - # a debuggability cost - for chunk in iterable: - fp.write(chunk) - - -def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, - allow_nan=False, cls=None, indent=None, separators=None, - encoding='utf-8', default=None, use_decimal=True, - namedtuple_as_object=True, tuple_as_array=True, - bigint_as_string=False, sort_keys=False, item_sort_key=None, - for_json=False, ignore_nan=False, int_as_string_bitcount=None, - iterable_as_array=False, **kw): - """Serialize ``obj`` to a JSON formatted ``str``. - - If ``skipkeys`` is true then ``dict`` keys that are not basic types - (``str``, ``int``, ``long``, ``float``, ``bool``, ``None``) - will be skipped instead of raising a ``TypeError``. - - If *ensure_ascii* is false (default: ``True``), then the output may - contain non-ASCII characters, so long as they do not need to be escaped - by JSON. When it is true, all non-ASCII characters are escaped. - - If ``check_circular`` is false, then the circular reference check - for container types will be skipped and a circular reference will - result in an ``OverflowError`` (or worse). - - If *allow_nan* is true (default: ``False``), then out of range ``float`` - values (``nan``, ``inf``, ``-inf``) will be serialized to - their JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``) - instead of raising a ValueError. See - *ignore_nan* for ECMA-262 compliant behavior. - - If ``indent`` is a string, then JSON array elements and object members - will be pretty-printed with a newline followed by that string repeated - for each level of nesting. ``None`` (the default) selects the most compact - representation without any newlines. For backwards compatibility with - versions of simplejson earlier than 2.1.0, an integer is also accepted - and is converted to a string with that many spaces. - - If specified, ``separators`` should be an - ``(item_separator, key_separator)`` tuple. The default is ``(', ', ': ')`` - if *indent* is ``None`` and ``(',', ': ')`` otherwise. To get the most - compact JSON representation, you should specify ``(',', ':')`` to eliminate - whitespace. - - ``encoding`` is the character encoding for bytes instances, default is - UTF-8. - - ``default(obj)`` is a function that should return a serializable version - of obj or raise TypeError. The default simply raises TypeError. - - If *use_decimal* is true (default: ``True``) then decimal.Decimal - will be natively serialized to JSON with full precision. - - If *namedtuple_as_object* is true (default: ``True``), - :class:`tuple` subclasses with ``_asdict()`` methods will be encoded - as JSON objects. - - If *tuple_as_array* is true (default: ``True``), - :class:`tuple` (and subclasses) will be encoded as JSON arrays. - - If *iterable_as_array* is true (default: ``False``), - any object not in the above table that implements ``__iter__()`` - will be encoded as a JSON array. - - If *bigint_as_string* is true (not the default), ints 2**53 and higher - or lower than -2**53 will be encoded as strings. This is to avoid the - rounding that happens in Javascript otherwise. - - If *int_as_string_bitcount* is a positive number (n), then int of size - greater than or equal to 2**n or lower than or equal to -2**n will be - encoded as strings. - - If specified, *item_sort_key* is a callable used to sort the items in - each dictionary. This is useful if you want to sort items other than - in alphabetical order by key. This option takes precedence over - *sort_keys*. - - If *sort_keys* is true (default: ``False``), the output of dictionaries - will be sorted by item. - - If *for_json* is true (default: ``False``), objects with a ``for_json()`` - method will use the return value of that method for encoding as JSON - instead of the object. - - If *ignore_nan* is true (default: ``False``), then out of range - :class:`float` values (``nan``, ``inf``, ``-inf``) will be serialized as - ``null`` in compliance with the ECMA-262 specification. If true, this will - override *allow_nan*. - - To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the - ``.default()`` method to serialize additional types), specify it with - the ``cls`` kwarg. NOTE: You should use *default* instead of subclassing - whenever possible. - - """ - # cached encoder - if (not skipkeys and ensure_ascii and - check_circular and not allow_nan and - cls is None and indent is None and separators is None and - encoding == 'utf-8' and default is None and use_decimal - and namedtuple_as_object and tuple_as_array and not iterable_as_array - and not bigint_as_string and not sort_keys - and not item_sort_key and not for_json - and not ignore_nan and int_as_string_bitcount is None - and not kw - ): - return _default_encoder.encode(obj) - if cls is None: - cls = JSONEncoder - return cls( - skipkeys=skipkeys, ensure_ascii=ensure_ascii, - check_circular=check_circular, allow_nan=allow_nan, indent=indent, - separators=separators, encoding=encoding, default=default, - use_decimal=use_decimal, - namedtuple_as_object=namedtuple_as_object, - tuple_as_array=tuple_as_array, - iterable_as_array=iterable_as_array, - bigint_as_string=bigint_as_string, - sort_keys=sort_keys, - item_sort_key=item_sort_key, - for_json=for_json, - ignore_nan=ignore_nan, - int_as_string_bitcount=int_as_string_bitcount, - **kw).encode(obj) - - -_default_decoder = JSONDecoder() - - -def load(fp, encoding=None, cls=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, object_pairs_hook=None, - use_decimal=False, allow_nan=False, **kw): - """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing - a JSON document as `str` or `bytes`) to a Python object. - - *encoding* determines the encoding used to interpret any - `bytes` objects decoded by this instance (``'utf-8'`` by - default). It has no effect when decoding `str` objects. - - *object_hook*, if specified, will be called with the result of every - JSON object decoded and its return value will be used in place of the - given :class:`dict`. This can be used to provide custom - deserializations (e.g. to support JSON-RPC class hinting). - - *object_pairs_hook* is an optional function that will be called with - the result of any object literal decode with an ordered list of pairs. - The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders - that rely on the order that the key and value pairs are decoded (for - example, :func:`collections.OrderedDict` will remember the order of - insertion). If *object_hook* is also defined, the *object_pairs_hook* - takes priority. - - *parse_float*, if specified, will be called with the string of every - JSON float to be decoded. By default, this is equivalent to - ``float(num_str)``. This can be used to use another datatype or parser - for JSON floats (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every - JSON int to be decoded. By default, this is equivalent to - ``int(num_str)``. This can be used to use another datatype or parser - for JSON integers (e.g. :class:`float`). - - *allow_nan*, if True (default false), will allow the parser to - accept the non-standard floats ``NaN``, ``Infinity``, and ``-Infinity`` - and enable the use of the deprecated *parse_constant*. - - If *use_decimal* is true (default: ``False``) then it implies - parse_float=decimal.Decimal for parity with ``dump``. - - *parse_constant*, if specified, will be - called with one of the following strings: ``'-Infinity'``, - ``'Infinity'``, ``'NaN'``. It is not recommended to use this feature, - as it is rare to parse non-compliant JSON containing these values. - - To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` - kwarg. NOTE: You should use *object_hook* or *object_pairs_hook* instead - of subclassing whenever possible. - - """ - return loads(fp.read(), - encoding=encoding, cls=cls, object_hook=object_hook, - parse_float=parse_float, parse_int=parse_int, - parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, - use_decimal=use_decimal, allow_nan=allow_nan, **kw) - - -def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, object_pairs_hook=None, - use_decimal=False, allow_nan=False, **kw): - """Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON - document) to a Python object. - - *encoding* determines the encoding used to interpret any - :class:`bytes` objects decoded by this instance (``'utf-8'`` by - default). It has no effect when decoding :class:`unicode` objects. - - *object_hook*, if specified, will be called with the result of every - JSON object decoded and its return value will be used in place of the - given :class:`dict`. This can be used to provide custom - deserializations (e.g. to support JSON-RPC class hinting). - - *object_pairs_hook* is an optional function that will be called with - the result of any object literal decode with an ordered list of pairs. - The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders - that rely on the order that the key and value pairs are decoded (for - example, :func:`collections.OrderedDict` will remember the order of - insertion). If *object_hook* is also defined, the *object_pairs_hook* - takes priority. - - *parse_float*, if specified, will be called with the string of every - JSON float to be decoded. By default, this is equivalent to - ``float(num_str)``. This can be used to use another datatype or parser - for JSON floats (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every - JSON int to be decoded. By default, this is equivalent to - ``int(num_str)``. This can be used to use another datatype or parser - for JSON integers (e.g. :class:`float`). - - *allow_nan*, if True (default false), will allow the parser to - accept the non-standard floats ``NaN``, ``Infinity``, and ``-Infinity`` - and enable the use of the deprecated *parse_constant*. - - If *use_decimal* is true (default: ``False``) then it implies - parse_float=decimal.Decimal for parity with ``dump``. - - *parse_constant*, if specified, will be - called with one of the following strings: ``'-Infinity'``, - ``'Infinity'``, ``'NaN'``. It is not recommended to use this feature, - as it is rare to parse non-compliant JSON containing these values. - - To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` - kwarg. NOTE: You should use *object_hook* or *object_pairs_hook* instead - of subclassing whenever possible. - - """ - if (cls is None and encoding is None and object_hook is None and - parse_int is None and parse_float is None and - parse_constant is None and object_pairs_hook is None - and not use_decimal and not allow_nan and not kw): - return _default_decoder.decode(s) - if cls is None: - cls = JSONDecoder - if object_hook is not None: - kw['object_hook'] = object_hook - if object_pairs_hook is not None: - kw['object_pairs_hook'] = object_pairs_hook - if parse_float is not None: - kw['parse_float'] = parse_float - if parse_int is not None: - kw['parse_int'] = parse_int - if parse_constant is not None: - kw['parse_constant'] = parse_constant - if use_decimal: - if parse_float is not None: - raise TypeError("use_decimal=True implies parse_float=Decimal") - kw['parse_float'] = Decimal - if allow_nan: - kw['allow_nan'] = True - return cls(encoding=encoding, **kw).decode(s) - - -def _toggle_speedups(enabled): - from . import decoder as dec - from . import encoder as enc - from . import scanner as scan - c_make_encoder = _import_c_make_encoder() - if enabled: - dec.scanstring = dec.c_scanstring or dec.py_scanstring - enc.c_make_encoder = c_make_encoder - enc.encode_basestring_ascii = (enc.c_encode_basestring_ascii or - enc.py_encode_basestring_ascii) - scan.make_scanner = scan.c_make_scanner or scan.py_make_scanner - else: - dec.scanstring = dec.py_scanstring - enc.c_make_encoder = None - enc.encode_basestring_ascii = enc.py_encode_basestring_ascii - scan.make_scanner = scan.py_make_scanner - dec.make_scanner = scan.make_scanner - global _default_decoder - _default_decoder = JSONDecoder() - global _default_encoder - _default_encoder = JSONEncoder() - -def simple_first(kv): - """Helper function to pass to item_sort_key to sort simple - elements to the top, then container elements. - """ - return (isinstance(kv[1], (list, dict, tuple)), kv[0]) diff --git a/contrib/python/simplejson/py3/simplejson/_speedups.c b/contrib/python/simplejson/py3/simplejson/_speedups.c deleted file mode 100644 index 4133b9bc761..00000000000 --- a/contrib/python/simplejson/py3/simplejson/_speedups.c +++ /dev/null @@ -1,3414 +0,0 @@ -/* -*- mode: C; c-file-style: "python"; c-basic-offset: 4 -*- */ -#include "Python.h" -#include "structmember.h" - -#if PY_MAJOR_VERSION >= 3 -#define PyInt_FromSsize_t PyLong_FromSsize_t -#define PyInt_AsSsize_t PyLong_AsSsize_t -#define PyInt_Check(obj) 0 -#define PyInt_CheckExact(obj) 0 -#define JSON_UNICHR Py_UCS4 -#define JSON_InternFromString PyUnicode_InternFromString -#define PyString_GET_SIZE PyUnicode_GET_LENGTH -#define JSON_StringCheck PyUnicode_Check -#define PY2_UNUSED -#define PY3_UNUSED UNUSED -#else /* PY_MAJOR_VERSION >= 3 */ -#define PY2_UNUSED UNUSED -#define PY3_UNUSED -#define JSON_StringCheck(obj) (PyString_Check(obj) || PyUnicode_Check(obj)) -#define PyBytes_Check PyString_Check -#define PyUnicode_READY(obj) 0 -#define PyUnicode_KIND(obj) (sizeof(Py_UNICODE)) -#define PyUnicode_DATA(obj) ((void *)(PyUnicode_AS_UNICODE(obj))) -#define PyUnicode_READ(kind, data, index) ((JSON_UNICHR)((const Py_UNICODE *)(data))[(index)]) -#define PyUnicode_GET_LENGTH PyUnicode_GET_SIZE -#define JSON_UNICHR Py_UNICODE -#define JSON_InternFromString PyString_InternFromString -#endif /* PY_MAJOR_VERSION < 3 */ - -#if PY_VERSION_HEX < 0x03090000 -#if !defined(PyObject_CallNoArgs) -#define PyObject_CallNoArgs(callable) PyObject_CallFunctionObjArgs(callable, NULL); -#endif -#if !defined(PyObject_CallOneArg) -#define PyObject_CallOneArg(callable, arg) PyObject_CallFunctionObjArgs(callable, arg, NULL); -#endif -#endif /* PY_VERSION_HEX < 0x03090000 */ - -#if PY_VERSION_HEX < 0x02070000 -#if !defined(PyOS_string_to_double) -#define PyOS_string_to_double json_PyOS_string_to_double -static double -json_PyOS_string_to_double(const char *s, char **endptr, PyObject *overflow_exception); -static double -json_PyOS_string_to_double(const char *s, char **endptr, PyObject *overflow_exception) -{ - double x; - assert(endptr == NULL); - assert(overflow_exception == NULL); - PyFPE_START_PROTECT("json_PyOS_string_to_double", return -1.0;) - x = PyOS_ascii_atof(s); - PyFPE_END_PROTECT(x) - return x; -} -#endif -#endif /* PY_VERSION_HEX < 0x02070000 */ - -#if PY_VERSION_HEX < 0x02060000 -#if !defined(Py_TYPE) -#define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) -#endif -#if !defined(Py_SIZE) -#define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) -#endif -#if !defined(PyVarObject_HEAD_INIT) -#define PyVarObject_HEAD_INIT(type, size) PyObject_HEAD_INIT(type) size, -#endif -#endif /* PY_VERSION_HEX < 0x02060000 */ - -#ifdef __GNUC__ -#define UNUSED __attribute__((__unused__)) -#else -#define UNUSED -#endif - -#define DEFAULT_ENCODING "utf-8" - -#define PyScanner_Check(op) PyObject_TypeCheck(op, &PyScannerType) -#define PyScanner_CheckExact(op) (Py_TYPE(op) == &PyScannerType) -#define PyEncoder_Check(op) PyObject_TypeCheck(op, &PyEncoderType) -#define PyEncoder_CheckExact(op) (Py_TYPE(op) == &PyEncoderType) - -#define JSON_ALLOW_NAN 1 -#define JSON_IGNORE_NAN 2 - -static PyObject *JSON_Infinity = NULL; -static PyObject *JSON_NegInfinity = NULL; -static PyObject *JSON_NaN = NULL; -static PyObject *JSON_EmptyUnicode = NULL; -#if PY_MAJOR_VERSION < 3 -static PyObject *JSON_EmptyStr = NULL; -#endif - -static PyTypeObject PyScannerType; -static PyTypeObject PyEncoderType; - -typedef struct { - PyObject *large_strings; /* A list of previously accumulated large strings */ - PyObject *small_strings; /* Pending small strings */ -} JSON_Accu; - -static int -JSON_Accu_Init(JSON_Accu *acc); -static int -JSON_Accu_Accumulate(JSON_Accu *acc, PyObject *unicode); -static PyObject * -JSON_Accu_FinishAsList(JSON_Accu *acc); -static void -JSON_Accu_Destroy(JSON_Accu *acc); - -#define ERR_EXPECTING_VALUE "Expecting value" -#define ERR_ARRAY_DELIMITER "Expecting ',' delimiter or ']'" -#define ERR_ARRAY_VALUE_FIRST "Expecting value or ']'" -#define ERR_OBJECT_DELIMITER "Expecting ',' delimiter or '}'" -#define ERR_OBJECT_PROPERTY "Expecting property name enclosed in double quotes" -#define ERR_OBJECT_PROPERTY_FIRST "Expecting property name enclosed in double quotes or '}'" -#define ERR_OBJECT_PROPERTY_DELIMITER "Expecting ':' delimiter" -#define ERR_STRING_UNTERMINATED "Unterminated string starting at" -#define ERR_STRING_CONTROL "Invalid control character %r at" -#define ERR_STRING_ESC1 "Invalid \\X escape sequence %r" -#define ERR_STRING_ESC4 "Invalid \\uXXXX escape sequence" -#define FOR_JSON_METHOD_NAME "for_json" -#define ASDICT_METHOD_NAME "_asdict" - - -typedef struct _PyScannerObject { - PyObject_HEAD - PyObject *encoding; - PyObject *strict_bool; - int strict; - PyObject *object_hook; - PyObject *pairs_hook; - PyObject *parse_float; - PyObject *parse_int; - PyObject *parse_constant; - PyObject *memo; -} PyScannerObject; - -static PyMemberDef scanner_members[] = { - {"encoding", T_OBJECT, offsetof(PyScannerObject, encoding), READONLY, "encoding"}, - {"strict", T_OBJECT, offsetof(PyScannerObject, strict_bool), READONLY, "strict"}, - {"object_hook", T_OBJECT, offsetof(PyScannerObject, object_hook), READONLY, "object_hook"}, - {"object_pairs_hook", T_OBJECT, offsetof(PyScannerObject, pairs_hook), READONLY, "object_pairs_hook"}, - {"parse_float", T_OBJECT, offsetof(PyScannerObject, parse_float), READONLY, "parse_float"}, - {"parse_int", T_OBJECT, offsetof(PyScannerObject, parse_int), READONLY, "parse_int"}, - {"parse_constant", T_OBJECT, offsetof(PyScannerObject, parse_constant), READONLY, "parse_constant"}, - {NULL} -}; - -typedef struct _PyEncoderObject { - PyObject_HEAD - PyObject *markers; - PyObject *defaultfn; - PyObject *encoder; - PyObject *indent; - PyObject *key_separator; - PyObject *item_separator; - PyObject *sort_keys; - PyObject *key_memo; - PyObject *encoding; - PyObject *Decimal; - PyObject *skipkeys_bool; - int skipkeys; - int fast_encode; - /* 0, JSON_ALLOW_NAN, JSON_IGNORE_NAN */ - int allow_or_ignore_nan; - int use_decimal; - int namedtuple_as_object; - int tuple_as_array; - int iterable_as_array; - PyObject *max_long_size; - PyObject *min_long_size; - PyObject *item_sort_key; - PyObject *item_sort_kw; - int for_json; -} PyEncoderObject; - -static PyMemberDef encoder_members[] = { - {"markers", T_OBJECT, offsetof(PyEncoderObject, markers), READONLY, "markers"}, - {"default", T_OBJECT, offsetof(PyEncoderObject, defaultfn), READONLY, "default"}, - {"encoder", T_OBJECT, offsetof(PyEncoderObject, encoder), READONLY, "encoder"}, - {"encoding", T_OBJECT, offsetof(PyEncoderObject, encoder), READONLY, "encoding"}, - {"indent", T_OBJECT, offsetof(PyEncoderObject, indent), READONLY, "indent"}, - {"key_separator", T_OBJECT, offsetof(PyEncoderObject, key_separator), READONLY, "key_separator"}, - {"item_separator", T_OBJECT, offsetof(PyEncoderObject, item_separator), READONLY, "item_separator"}, - {"sort_keys", T_OBJECT, offsetof(PyEncoderObject, sort_keys), READONLY, "sort_keys"}, - /* Python 2.5 does not support T_BOOl */ - {"skipkeys", T_OBJECT, offsetof(PyEncoderObject, skipkeys_bool), READONLY, "skipkeys"}, - {"key_memo", T_OBJECT, offsetof(PyEncoderObject, key_memo), READONLY, "key_memo"}, - {"item_sort_key", T_OBJECT, offsetof(PyEncoderObject, item_sort_key), READONLY, "item_sort_key"}, - {"max_long_size", T_OBJECT, offsetof(PyEncoderObject, max_long_size), READONLY, "max_long_size"}, - {"min_long_size", T_OBJECT, offsetof(PyEncoderObject, min_long_size), READONLY, "min_long_size"}, - {NULL} -}; - -static PyObject * -join_list_unicode(PyObject *lst); -static PyObject * -JSON_ParseEncoding(PyObject *encoding); -static PyObject * -maybe_quote_bigint(PyEncoderObject* s, PyObject *encoded, PyObject *obj); -static Py_ssize_t -ascii_char_size(JSON_UNICHR c); -static Py_ssize_t -ascii_escape_char(JSON_UNICHR c, char *output, Py_ssize_t chars); -static PyObject * -ascii_escape_unicode(PyObject *pystr); -static PyObject * -ascii_escape_str(PyObject *pystr); -static PyObject * -py_encode_basestring_ascii(PyObject* self UNUSED, PyObject *pystr); -#if PY_MAJOR_VERSION < 3 -static PyObject * -join_list_string(PyObject *lst); -static PyObject * -scan_once_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr); -static PyObject * -scanstring_str(PyObject *pystr, Py_ssize_t end, char *encoding, int strict, Py_ssize_t *next_end_ptr); -static PyObject * -_parse_object_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr); -#endif -static PyObject * -scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next_end_ptr); -static PyObject * -scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr); -static PyObject * -_build_rval_index_tuple(PyObject *rval, Py_ssize_t idx); -static PyObject * -scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds); -static void -scanner_dealloc(PyObject *self); -static int -scanner_clear(PyObject *self); -static PyObject * -encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds); -static void -encoder_dealloc(PyObject *self); -static int -encoder_clear(PyObject *self); -static int -is_raw_json(PyObject *obj); -static PyObject * -encoder_stringify_key(PyEncoderObject *s, PyObject *key); -static int -encoder_listencode_list(PyEncoderObject *s, JSON_Accu *rval, PyObject *seq, Py_ssize_t indent_level); -static int -encoder_listencode_obj(PyEncoderObject *s, JSON_Accu *rval, PyObject *obj, Py_ssize_t indent_level); -static int -encoder_listencode_dict(PyEncoderObject *s, JSON_Accu *rval, PyObject *dct, Py_ssize_t indent_level); -static PyObject * -_encoded_const(PyObject *obj); -static void -raise_errmsg(char *msg, PyObject *s, Py_ssize_t end); -static PyObject * -encoder_encode_string(PyEncoderObject *s, PyObject *obj); -static int -_convertPyInt_AsSsize_t(PyObject *o, Py_ssize_t *size_ptr); -static PyObject * -_convertPyInt_FromSsize_t(Py_ssize_t *size_ptr); -static int -_call_json_method(PyObject *obj, const char *method_name, PyObject **result); -static PyObject * -encoder_encode_float(PyEncoderObject *s, PyObject *obj); -static PyObject * -moduleinit(void); - -#define S_CHAR(c) (c >= ' ' && c <= '~' && c != '\\' && c != '"') -#define IS_WHITESPACE(c) (((c) == ' ') || ((c) == '\t') || ((c) == '\n') || ((c) == '\r')) - -#define MIN_EXPANSION 6 - -static PyObject* RawJSONType = NULL; -static int -is_raw_json(PyObject *obj) -{ - return PyObject_IsInstance(obj, RawJSONType) ? 1 : 0; -} - -static int -JSON_Accu_Init(JSON_Accu *acc) -{ - /* Lazily allocated */ - acc->large_strings = NULL; - acc->small_strings = PyList_New(0); - if (acc->small_strings == NULL) - return -1; - return 0; -} - -static int -flush_accumulator(JSON_Accu *acc) -{ - Py_ssize_t nsmall = PyList_GET_SIZE(acc->small_strings); - if (nsmall) { - int ret; - PyObject *joined; - if (acc->large_strings == NULL) { - acc->large_strings = PyList_New(0); - if (acc->large_strings == NULL) - return -1; - } -#if PY_MAJOR_VERSION >= 3 - joined = join_list_unicode(acc->small_strings); -#else /* PY_MAJOR_VERSION >= 3 */ - joined = join_list_string(acc->small_strings); -#endif /* PY_MAJOR_VERSION < 3 */ - if (joined == NULL) - return -1; - if (PyList_SetSlice(acc->small_strings, 0, nsmall, NULL)) { - Py_DECREF(joined); - return -1; - } - ret = PyList_Append(acc->large_strings, joined); - Py_DECREF(joined); - return ret; - } - return 0; -} - -static int -JSON_Accu_Accumulate(JSON_Accu *acc, PyObject *unicode) -{ - Py_ssize_t nsmall; -#if PY_MAJOR_VERSION >= 3 - assert(PyUnicode_Check(unicode)); -#else /* PY_MAJOR_VERSION >= 3 */ - assert(PyString_Check(unicode) || PyUnicode_Check(unicode)); -#endif /* PY_MAJOR_VERSION < 3 */ - - if (PyList_Append(acc->small_strings, unicode)) - return -1; - nsmall = PyList_GET_SIZE(acc->small_strings); - /* Each item in a list of unicode objects has an overhead (in 64-bit - * builds) of: - * - 8 bytes for the list slot - * - 56 bytes for the header of the unicode object - * that is, 64 bytes. 100000 such objects waste more than 6MB - * compared to a single concatenated string. - */ - if (nsmall < 100000) - return 0; - return flush_accumulator(acc); -} - -static PyObject * -JSON_Accu_FinishAsList(JSON_Accu *acc) -{ - int ret; - PyObject *res; - - ret = flush_accumulator(acc); - Py_CLEAR(acc->small_strings); - if (ret) { - Py_CLEAR(acc->large_strings); - return NULL; - } - res = acc->large_strings; - acc->large_strings = NULL; - if (res == NULL) - return PyList_New(0); - return res; -} - -static void -JSON_Accu_Destroy(JSON_Accu *acc) -{ - Py_CLEAR(acc->small_strings); - Py_CLEAR(acc->large_strings); -} - -static int -IS_DIGIT(JSON_UNICHR c) -{ - return c >= '0' && c <= '9'; -} - -static PyObject * -maybe_quote_bigint(PyEncoderObject* s, PyObject *encoded, PyObject *obj) -{ - if (s->max_long_size != Py_None && s->min_long_size != Py_None) { - if (PyObject_RichCompareBool(obj, s->max_long_size, Py_GE) || - PyObject_RichCompareBool(obj, s->min_long_size, Py_LE)) { -#if PY_MAJOR_VERSION >= 3 - PyObject* quoted = PyUnicode_FromFormat("\"%U\"", encoded); -#else - PyObject* quoted = PyString_FromFormat("\"%s\"", - PyString_AsString(encoded)); -#endif - Py_DECREF(encoded); - encoded = quoted; - } - } - - return encoded; -} - -static int -_call_json_method(PyObject *obj, const char *method_name, PyObject **result) -{ - int rval = 0; - PyObject *method = PyObject_GetAttrString(obj, method_name); - if (method == NULL) { - PyErr_Clear(); - return 0; - } - if (PyCallable_Check(method)) { - PyObject *tmp = PyObject_CallNoArgs(method); - if (tmp == NULL && PyErr_ExceptionMatches(PyExc_TypeError)) { - PyErr_Clear(); - } else { - // This will set result to NULL if a TypeError occurred, - // which must be checked by the caller - *result = tmp; - rval = 1; - } - } - Py_DECREF(method); - return rval; -} - -static int -_convertPyInt_AsSsize_t(PyObject *o, Py_ssize_t *size_ptr) -{ - /* PyObject to Py_ssize_t converter */ - *size_ptr = PyInt_AsSsize_t(o); - if (*size_ptr == -1 && PyErr_Occurred()) - return 0; - return 1; -} - -static PyObject * -_convertPyInt_FromSsize_t(Py_ssize_t *size_ptr) -{ - /* Py_ssize_t to PyObject converter */ - return PyInt_FromSsize_t(*size_ptr); -} - -static Py_ssize_t -ascii_escape_char(JSON_UNICHR c, char *output, Py_ssize_t chars) -{ - /* Escape unicode code point c to ASCII escape sequences - in char *output. output must have at least 12 bytes unused to - accommodate an escaped surrogate pair "\uXXXX\uXXXX" */ - if (S_CHAR(c)) { - output[chars++] = (char)c; - } - else { - output[chars++] = '\\'; - switch (c) { - case '\\': output[chars++] = (char)c; break; - case '"': output[chars++] = (char)c; break; - case '\b': output[chars++] = 'b'; break; - case '\f': output[chars++] = 'f'; break; - case '\n': output[chars++] = 'n'; break; - case '\r': output[chars++] = 'r'; break; - case '\t': output[chars++] = 't'; break; - default: -#if PY_MAJOR_VERSION >= 3 || defined(Py_UNICODE_WIDE) - if (c >= 0x10000) { - /* UTF-16 surrogate pair */ - JSON_UNICHR v = c - 0x10000; - c = 0xd800 | ((v >> 10) & 0x3ff); - output[chars++] = 'u'; - output[chars++] = "0123456789abcdef"[(c >> 12) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 8) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 4) & 0xf]; - output[chars++] = "0123456789abcdef"[(c ) & 0xf]; - c = 0xdc00 | (v & 0x3ff); - output[chars++] = '\\'; - } -#endif - output[chars++] = 'u'; - output[chars++] = "0123456789abcdef"[(c >> 12) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 8) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 4) & 0xf]; - output[chars++] = "0123456789abcdef"[(c ) & 0xf]; - } - } - return chars; -} - -static Py_ssize_t -ascii_char_size(JSON_UNICHR c) -{ - if (S_CHAR(c)) { - return 1; - } - else if (c == '\\' || - c == '"' || - c == '\b' || - c == '\f' || - c == '\n' || - c == '\r' || - c == '\t') { - return 2; - } -#if PY_MAJOR_VERSION >= 3 || defined(Py_UNICODE_WIDE) - else if (c >= 0x10000U) { - return 2 * MIN_EXPANSION; - } -#endif - else { - return MIN_EXPANSION; - } -} - -static PyObject * -ascii_escape_unicode(PyObject *pystr) -{ - /* Take a PyUnicode pystr and return a new ASCII-only escaped PyString */ - Py_ssize_t i; - Py_ssize_t input_chars = PyUnicode_GET_LENGTH(pystr); - Py_ssize_t output_size = 2; - Py_ssize_t chars; - PY2_UNUSED int kind = PyUnicode_KIND(pystr); - void *data = PyUnicode_DATA(pystr); - PyObject *rval; - char *output; - - output_size = 2; - for (i = 0; i < input_chars; i++) { - output_size += ascii_char_size(PyUnicode_READ(kind, data, i)); - } -#if PY_MAJOR_VERSION >= 3 - rval = PyUnicode_New(output_size, 127); - if (rval == NULL) { - return NULL; - } - assert(PyUnicode_KIND(rval) == PyUnicode_1BYTE_KIND); - output = (char *)PyUnicode_DATA(rval); -#else - rval = PyString_FromStringAndSize(NULL, output_size); - if (rval == NULL) { - return NULL; - } - output = PyString_AS_STRING(rval); -#endif - chars = 0; - output[chars++] = '"'; - for (i = 0; i < input_chars; i++) { - chars = ascii_escape_char(PyUnicode_READ(kind, data, i), output, chars); - } - output[chars++] = '"'; - assert(chars == output_size); - return rval; -} - -#if PY_MAJOR_VERSION >= 3 - -static PyObject * -ascii_escape_str(PyObject *pystr) -{ - PyObject *rval; - PyObject *input = PyUnicode_DecodeUTF8(PyBytes_AS_STRING(pystr), PyBytes_GET_SIZE(pystr), NULL); - if (input == NULL) - return NULL; - rval = ascii_escape_unicode(input); - Py_DECREF(input); - return rval; -} - -#else /* PY_MAJOR_VERSION >= 3 */ - -static PyObject * -ascii_escape_str(PyObject *pystr) -{ - /* Take a PyString pystr and return a new ASCII-only escaped PyString */ - Py_ssize_t i; - Py_ssize_t input_chars; - Py_ssize_t output_size; - Py_ssize_t chars; - PyObject *rval; - char *output; - char *input_str; - - input_chars = PyString_GET_SIZE(pystr); - input_str = PyString_AS_STRING(pystr); - output_size = 2; - - /* Fast path for a string that's already ASCII */ - for (i = 0; i < input_chars; i++) { - JSON_UNICHR c = (JSON_UNICHR)input_str[i]; - if (c > 0x7f) { - /* We hit a non-ASCII character, bail to unicode mode */ - PyObject *uni; - uni = PyUnicode_DecodeUTF8(input_str, input_chars, "strict"); - if (uni == NULL) { - return NULL; - } - rval = ascii_escape_unicode(uni); - Py_DECREF(uni); - return rval; - } - output_size += ascii_char_size(c); - } - - rval = PyString_FromStringAndSize(NULL, output_size); - if (rval == NULL) { - return NULL; - } - chars = 0; - output = PyString_AS_STRING(rval); - output[chars++] = '"'; - for (i = 0; i < input_chars; i++) { - chars = ascii_escape_char((JSON_UNICHR)input_str[i], output, chars); - } - output[chars++] = '"'; - assert(chars == output_size); - return rval; -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -encoder_stringify_key(PyEncoderObject *s, PyObject *key) -{ - if (PyUnicode_Check(key)) { - Py_INCREF(key); - return key; - } -#if PY_MAJOR_VERSION >= 3 - else if (PyBytes_Check(key) && s->encoding != NULL) { - const char *encoding = PyUnicode_AsUTF8(s->encoding); - if (encoding == NULL) - return NULL; - return PyUnicode_Decode( - PyBytes_AS_STRING(key), - PyBytes_GET_SIZE(key), - encoding, - NULL); - } -#else /* PY_MAJOR_VERSION >= 3 */ - else if (PyString_Check(key)) { - Py_INCREF(key); - return key; - } -#endif /* PY_MAJOR_VERSION < 3 */ - else if (PyFloat_Check(key)) { - return encoder_encode_float(s, key); - } - else if (key == Py_True || key == Py_False || key == Py_None) { - /* This must come before the PyInt_Check because - True and False are also 1 and 0.*/ - return _encoded_const(key); - } - else if (PyInt_Check(key) || PyLong_Check(key)) { - if (!(PyInt_CheckExact(key) || PyLong_CheckExact(key))) { - /* See #118, do not trust custom str/repr */ - PyObject *res; - PyObject *tmp = PyObject_CallOneArg((PyObject *)&PyLong_Type, key); - if (tmp == NULL) { - return NULL; - } - res = PyObject_Str(tmp); - Py_DECREF(tmp); - return res; - } - else { - return PyObject_Str(key); - } - } - else if (s->use_decimal && PyObject_TypeCheck(key, (PyTypeObject *)s->Decimal)) { - return PyObject_Str(key); - } - if (s->skipkeys) { - Py_INCREF(Py_None); - return Py_None; - } - PyErr_Format(PyExc_TypeError, - "keys must be str, int, float, bool or None, " - "not %.100s", key->ob_type->tp_name); - return NULL; -} - -static PyObject * -encoder_dict_iteritems(PyEncoderObject *s, PyObject *dct) -{ - PyObject *items; - PyObject *iter = NULL; - PyObject *lst = NULL; - PyObject *item = NULL; - PyObject *kstr = NULL; - PyObject *sortfun = NULL; - PyObject *sortres; - static PyObject *sortargs = NULL; - - if (sortargs == NULL) { - sortargs = PyTuple_New(0); - if (sortargs == NULL) - return NULL; - } - - if (PyDict_CheckExact(dct)) - items = PyDict_Items(dct); - else - items = PyMapping_Items(dct); - if (items == NULL) - return NULL; - iter = PyObject_GetIter(items); - Py_DECREF(items); - if (iter == NULL) - return NULL; - if (s->item_sort_kw == Py_None) - return iter; - lst = PyList_New(0); - if (lst == NULL) - goto bail; - while ((item = PyIter_Next(iter))) { - PyObject *key, *value; - if (!PyTuple_Check(item) || Py_SIZE(item) != 2) { - PyErr_SetString(PyExc_ValueError, "items must return 2-tuples"); - goto bail; - } - key = PyTuple_GET_ITEM(item, 0); - if (key == NULL) - goto bail; -#if PY_MAJOR_VERSION < 3 - else if (PyString_Check(key)) { - /* item can be added as-is */ - } -#endif /* PY_MAJOR_VERSION < 3 */ - else if (PyUnicode_Check(key)) { - /* item can be added as-is */ - } - else { - PyObject *tpl; - kstr = encoder_stringify_key(s, key); - if (kstr == NULL) - goto bail; - else if (kstr == Py_None) { - /* skipkeys */ - Py_DECREF(kstr); - continue; - } - value = PyTuple_GET_ITEM(item, 1); - if (value == NULL) - goto bail; - tpl = PyTuple_Pack(2, kstr, value); - if (tpl == NULL) - goto bail; - Py_CLEAR(kstr); - Py_DECREF(item); - item = tpl; - } - if (PyList_Append(lst, item)) - goto bail; - Py_DECREF(item); - } - Py_CLEAR(iter); - if (PyErr_Occurred()) - goto bail; - sortfun = PyObject_GetAttrString(lst, "sort"); - if (sortfun == NULL) - goto bail; - sortres = PyObject_Call(sortfun, sortargs, s->item_sort_kw); - if (!sortres) - goto bail; - Py_DECREF(sortres); - Py_CLEAR(sortfun); - iter = PyObject_GetIter(lst); - Py_CLEAR(lst); - return iter; -bail: - Py_XDECREF(sortfun); - Py_XDECREF(kstr); - Py_XDECREF(item); - Py_XDECREF(lst); - Py_XDECREF(iter); - return NULL; -} - -/* Use JSONDecodeError exception to raise a nice looking ValueError subclass */ -static PyObject *JSONDecodeError = NULL; -static void -raise_errmsg(char *msg, PyObject *s, Py_ssize_t end) -{ - PyObject *exc = PyObject_CallFunction(JSONDecodeError, "(zOO&)", msg, s, _convertPyInt_FromSsize_t, &end); - if (exc) { - PyErr_SetObject(JSONDecodeError, exc); - Py_DECREF(exc); - } -} - -static PyObject * -join_list_unicode(PyObject *lst) -{ - /* return u''.join(lst) */ - return PyUnicode_Join(JSON_EmptyUnicode, lst); -} - -#if PY_MAJOR_VERSION >= 3 -#define join_list_string join_list_unicode -#else /* PY_MAJOR_VERSION >= 3 */ -static PyObject * -join_list_string(PyObject *lst) -{ - /* return ''.join(lst) */ - static PyObject *joinfn = NULL; - if (joinfn == NULL) { - joinfn = PyObject_GetAttrString(JSON_EmptyStr, "join"); - if (joinfn == NULL) - return NULL; - } - return PyObject_CallOneArg(joinfn, lst); -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -_build_rval_index_tuple(PyObject *rval, Py_ssize_t idx) -{ - /* return (rval, idx) tuple, stealing reference to rval */ - PyObject *tpl; - PyObject *pyidx; - /* - steal a reference to rval, returns (rval, idx) - */ - if (rval == NULL) { - assert(PyErr_Occurred()); - return NULL; - } - pyidx = PyInt_FromSsize_t(idx); - if (pyidx == NULL) { - Py_DECREF(rval); - return NULL; - } - tpl = PyTuple_New(2); - if (tpl == NULL) { - Py_DECREF(pyidx); - Py_DECREF(rval); - return NULL; - } - PyTuple_SET_ITEM(tpl, 0, rval); - PyTuple_SET_ITEM(tpl, 1, pyidx); - return tpl; -} - -#define APPEND_OLD_CHUNK \ - if (chunk != NULL) { \ - if (chunks == NULL) { \ - chunks = PyList_New(0); \ - if (chunks == NULL) { \ - goto bail; \ - } \ - } \ - if (PyList_Append(chunks, chunk)) { \ - goto bail; \ - } \ - Py_CLEAR(chunk); \ - } - -#if PY_MAJOR_VERSION < 3 -static PyObject * -scanstring_str(PyObject *pystr, Py_ssize_t end, char *encoding, int strict, Py_ssize_t *next_end_ptr) -{ - /* Read the JSON string from PyString pystr. - end is the index of the first character after the quote. - encoding is the encoding of pystr (must be an ASCII superset) - if strict is zero then literal control characters are allowed - *next_end_ptr is a return-by-reference index of the character - after the end quote - - Return value is a new PyString (if ASCII-only) or PyUnicode - */ - PyObject *rval; - Py_ssize_t len = PyString_GET_SIZE(pystr); - Py_ssize_t begin = end - 1; - Py_ssize_t next = begin; - int has_unicode = 0; - char *buf = PyString_AS_STRING(pystr); - PyObject *chunks = NULL; - PyObject *chunk = NULL; - PyObject *strchunk = NULL; - - if (len == end) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - else if (end < 0 || len < end) { - PyErr_SetString(PyExc_ValueError, "end is out of bounds"); - goto bail; - } - while (1) { - /* Find the end of the string or the next escape */ - Py_UNICODE c = 0; - for (next = end; next < len; next++) { - c = (unsigned char)buf[next]; - if (c == '"' || c == '\\') { - break; - } - else if (strict && c <= 0x1f) { - raise_errmsg(ERR_STRING_CONTROL, pystr, next); - goto bail; - } - else if (c > 0x7f) { - has_unicode = 1; - } - } - if (!(c == '"' || c == '\\')) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - /* Pick up this chunk if it's not zero length */ - if (next != end) { - APPEND_OLD_CHUNK - strchunk = PyString_FromStringAndSize(&buf[end], next - end); - if (strchunk == NULL) { - goto bail; - } - if (has_unicode) { - chunk = PyUnicode_FromEncodedObject(strchunk, encoding, NULL); - Py_DECREF(strchunk); - if (chunk == NULL) { - goto bail; - } - } - else { - chunk = strchunk; - } - } - next++; - if (c == '"') { - end = next; - break; - } - if (next == len) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - c = buf[next]; - if (c != 'u') { - /* Non-unicode backslash escapes */ - end = next + 1; - switch (c) { - case '"': break; - case '\\': break; - case '/': break; - case 'b': c = '\b'; break; - case 'f': c = '\f'; break; - case 'n': c = '\n'; break; - case 'r': c = '\r'; break; - case 't': c = '\t'; break; - default: c = 0; - } - if (c == 0) { - raise_errmsg(ERR_STRING_ESC1, pystr, end - 2); - goto bail; - } - } - else { - c = 0; - next++; - end = next + 4; - if (end >= len) { - raise_errmsg(ERR_STRING_ESC4, pystr, next - 1); - goto bail; - } - /* Decode 4 hex digits */ - for (; next < end; next++) { - JSON_UNICHR digit = (JSON_UNICHR)buf[next]; - c <<= 4; - switch (digit) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - c |= (digit - '0'); break; - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': - c |= (digit - 'a' + 10); break; - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': - c |= (digit - 'A' + 10); break; - default: - raise_errmsg(ERR_STRING_ESC4, pystr, end - 5); - goto bail; - } - } -#if defined(Py_UNICODE_WIDE) - /* Surrogate pair */ - if ((c & 0xfc00) == 0xd800) { - if (end + 6 < len && buf[next] == '\\' && buf[next+1] == 'u') { - JSON_UNICHR c2 = 0; - end += 6; - /* Decode 4 hex digits */ - for (next += 2; next < end; next++) { - c2 <<= 4; - JSON_UNICHR digit = buf[next]; - switch (digit) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - c2 |= (digit - '0'); break; - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': - c2 |= (digit - 'a' + 10); break; - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': - c2 |= (digit - 'A' + 10); break; - default: - raise_errmsg(ERR_STRING_ESC4, pystr, end - 5); - goto bail; - } - } - if ((c2 & 0xfc00) != 0xdc00) { - /* not a low surrogate, rewind */ - end -= 6; - next = end; - } - else { - c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00)); - } - } - } -#endif /* Py_UNICODE_WIDE */ - } - if (c > 0x7f) { - has_unicode = 1; - } - APPEND_OLD_CHUNK - if (has_unicode) { - chunk = PyUnicode_FromOrdinal(c); - if (chunk == NULL) { - goto bail; - } - } - else { - char c_char = Py_CHARMASK(c); - chunk = PyString_FromStringAndSize(&c_char, 1); - if (chunk == NULL) { - goto bail; - } - } - } - - if (chunks == NULL) { - if (chunk != NULL) - rval = chunk; - else { - rval = JSON_EmptyStr; - Py_INCREF(rval); - } - } - else { - APPEND_OLD_CHUNK - rval = join_list_string(chunks); - if (rval == NULL) { - goto bail; - } - Py_CLEAR(chunks); - } - - *next_end_ptr = end; - return rval; -bail: - *next_end_ptr = -1; - Py_XDECREF(chunk); - Py_XDECREF(chunks); - return NULL; -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next_end_ptr) -{ - /* Read the JSON string from PyUnicode pystr. - end is the index of the first character after the quote. - if strict is zero then literal control characters are allowed - *next_end_ptr is a return-by-reference index of the character - after the end quote - - Return value is a new PyUnicode - */ - PyObject *rval; - Py_ssize_t begin = end - 1; - Py_ssize_t next = begin; - PY2_UNUSED int kind = PyUnicode_KIND(pystr); - Py_ssize_t len = PyUnicode_GET_LENGTH(pystr); - void *buf = PyUnicode_DATA(pystr); - PyObject *chunks = NULL; - PyObject *chunk = NULL; - - if (len == end) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - else if (end < 0 || len < end) { - PyErr_SetString(PyExc_ValueError, "end is out of bounds"); - goto bail; - } - while (1) { - /* Find the end of the string or the next escape */ - JSON_UNICHR c = 0; - for (next = end; next < len; next++) { - c = PyUnicode_READ(kind, buf, next); - if (c == '"' || c == '\\') { - break; - } - else if (strict && c <= 0x1f) { - raise_errmsg(ERR_STRING_CONTROL, pystr, next); - goto bail; - } - } - if (!(c == '"' || c == '\\')) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - /* Pick up this chunk if it's not zero length */ - if (next != end) { - APPEND_OLD_CHUNK -#if PY_MAJOR_VERSION < 3 - chunk = PyUnicode_FromUnicode(&((const Py_UNICODE *)buf)[end], next - end); -#else - chunk = PyUnicode_Substring(pystr, end, next); -#endif - if (chunk == NULL) { - goto bail; - } - } - next++; - if (c == '"') { - end = next; - break; - } - if (next == len) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - c = PyUnicode_READ(kind, buf, next); - if (c != 'u') { - /* Non-unicode backslash escapes */ - end = next + 1; - switch (c) { - case '"': break; - case '\\': break; - case '/': break; - case 'b': c = '\b'; break; - case 'f': c = '\f'; break; - case 'n': c = '\n'; break; - case 'r': c = '\r'; break; - case 't': c = '\t'; break; - default: c = 0; - } - if (c == 0) { - raise_errmsg(ERR_STRING_ESC1, pystr, end - 2); - goto bail; - } - } - else { - c = 0; - next++; - end = next + 4; - if (end >= len) { - raise_errmsg(ERR_STRING_ESC4, pystr, next - 1); - goto bail; - } - /* Decode 4 hex digits */ - for (; next < end; next++) { - JSON_UNICHR digit = PyUnicode_READ(kind, buf, next); - c <<= 4; - switch (digit) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - c |= (digit - '0'); break; - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': - c |= (digit - 'a' + 10); break; - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': - c |= (digit - 'A' + 10); break; - default: - raise_errmsg(ERR_STRING_ESC4, pystr, end - 5); - goto bail; - } - } -#if PY_MAJOR_VERSION >= 3 || defined(Py_UNICODE_WIDE) - /* Surrogate pair */ - if ((c & 0xfc00) == 0xd800) { - JSON_UNICHR c2 = 0; - if (end + 6 < len && - PyUnicode_READ(kind, buf, next) == '\\' && - PyUnicode_READ(kind, buf, next + 1) == 'u') { - end += 6; - /* Decode 4 hex digits */ - for (next += 2; next < end; next++) { - JSON_UNICHR digit = PyUnicode_READ(kind, buf, next); - c2 <<= 4; - switch (digit) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - c2 |= (digit - '0'); break; - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': - c2 |= (digit - 'a' + 10); break; - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': - c2 |= (digit - 'A' + 10); break; - default: - raise_errmsg(ERR_STRING_ESC4, pystr, end - 5); - goto bail; - } - } - if ((c2 & 0xfc00) != 0xdc00) { - /* not a low surrogate, rewind */ - end -= 6; - next = end; - } - else { - c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00)); - } - } - } -#endif - } - APPEND_OLD_CHUNK - chunk = PyUnicode_FromOrdinal(c); - if (chunk == NULL) { - goto bail; - } - } - - if (chunks == NULL) { - if (chunk != NULL) - rval = chunk; - else { - rval = JSON_EmptyUnicode; - Py_INCREF(rval); - } - } - else { - APPEND_OLD_CHUNK - rval = join_list_unicode(chunks); - if (rval == NULL) { - goto bail; - } - Py_CLEAR(chunks); - } - *next_end_ptr = end; - return rval; -bail: - *next_end_ptr = -1; - Py_XDECREF(chunk); - Py_XDECREF(chunks); - return NULL; -} - -PyDoc_STRVAR(pydoc_scanstring, - "scanstring(basestring, end, encoding, strict=True) -> (str, end)\n" - "\n" - "Scan the string s for a JSON string. End is the index of the\n" - "character in s after the quote that started the JSON string.\n" - "Unescapes all valid JSON string escape sequences and raises ValueError\n" - "on attempt to decode an invalid string. If strict is False then literal\n" - "control characters are allowed in the string.\n" - "\n" - "Returns a tuple of the decoded string and the index of the character in s\n" - "after the end quote." -); - -static PyObject * -py_scanstring(PyObject* self UNUSED, PyObject *args) -{ - PyObject *pystr; - PyObject *rval; - Py_ssize_t end; - Py_ssize_t next_end = -1; - char *encoding = NULL; - int strict = 1; - if (!PyArg_ParseTuple(args, "OO&|zi:scanstring", &pystr, _convertPyInt_AsSsize_t, &end, &encoding, &strict)) { - return NULL; - } - if (encoding == NULL) { - encoding = DEFAULT_ENCODING; - } - if (PyUnicode_Check(pystr)) { - if (PyUnicode_READY(pystr)) - return NULL; - rval = scanstring_unicode(pystr, end, strict, &next_end); - } -#if PY_MAJOR_VERSION < 3 - /* Using a bytes input is unsupported for scanning in Python 3. - It is coerced to str in the decoder before it gets here. */ - else if (PyString_Check(pystr)) { - rval = scanstring_str(pystr, end, encoding, strict, &next_end); - } -#endif - else { - PyErr_Format(PyExc_TypeError, - "first argument must be a string, not %.80s", - Py_TYPE(pystr)->tp_name); - return NULL; - } - return _build_rval_index_tuple(rval, next_end); -} - -PyDoc_STRVAR(pydoc_encode_basestring_ascii, - "encode_basestring_ascii(basestring) -> str\n" - "\n" - "Return an ASCII-only JSON representation of a Python string" -); - -static PyObject * -py_encode_basestring_ascii(PyObject* self UNUSED, PyObject *pystr) -{ - /* Return an ASCII-only JSON representation of a Python string */ - /* METH_O */ - if (PyBytes_Check(pystr)) { - return ascii_escape_str(pystr); - } - else if (PyUnicode_Check(pystr)) { - if (PyUnicode_READY(pystr)) - return NULL; - return ascii_escape_unicode(pystr); - } - else { - PyErr_Format(PyExc_TypeError, - "first argument must be a string, not %.80s", - Py_TYPE(pystr)->tp_name); - return NULL; - } -} - -static void -scanner_dealloc(PyObject *self) -{ - /* bpo-31095: UnTrack is needed before calling any callbacks */ - PyObject_GC_UnTrack(self); - scanner_clear(self); - Py_TYPE(self)->tp_free(self); -} - -static int -scanner_traverse(PyObject *self, visitproc visit, void *arg) -{ - PyScannerObject *s; - assert(PyScanner_Check(self)); - s = (PyScannerObject *)self; - Py_VISIT(s->encoding); - Py_VISIT(s->strict_bool); - Py_VISIT(s->object_hook); - Py_VISIT(s->pairs_hook); - Py_VISIT(s->parse_float); - Py_VISIT(s->parse_int); - Py_VISIT(s->parse_constant); - Py_VISIT(s->memo); - return 0; -} - -static int -scanner_clear(PyObject *self) -{ - PyScannerObject *s; - assert(PyScanner_Check(self)); - s = (PyScannerObject *)self; - Py_CLEAR(s->encoding); - Py_CLEAR(s->strict_bool); - Py_CLEAR(s->object_hook); - Py_CLEAR(s->pairs_hook); - Py_CLEAR(s->parse_float); - Py_CLEAR(s->parse_int); - Py_CLEAR(s->parse_constant); - Py_CLEAR(s->memo); - return 0; -} - -#if PY_MAJOR_VERSION < 3 -static PyObject * -_parse_object_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON object from PyString pystr. - idx is the index of the first character after the opening curly brace. - *next_idx_ptr is a return-by-reference index to the first character after - the closing curly brace. - - Returns a new PyObject (usually a dict, but object_hook or - object_pairs_hook can change that) - */ - char *str = PyString_AS_STRING(pystr); - Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1; - PyObject *rval = NULL; - PyObject *pairs = NULL; - PyObject *item; - PyObject *key = NULL; - PyObject *val = NULL; - char *encoding = PyString_AS_STRING(s->encoding); - int has_pairs_hook = (s->pairs_hook != Py_None); - int did_parse = 0; - Py_ssize_t next_idx; - if (has_pairs_hook) { - pairs = PyList_New(0); - if (pairs == NULL) - return NULL; - } - else { - rval = PyDict_New(); - if (rval == NULL) - return NULL; - } - - /* skip whitespace after { */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* only loop if the object is non-empty */ - if (idx <= end_idx && str[idx] != '}') { - int trailing_delimiter = 0; - while (idx <= end_idx) { - PyObject *memokey; - trailing_delimiter = 0; - - /* read key */ - if (str[idx] != '"') { - raise_errmsg(ERR_OBJECT_PROPERTY, pystr, idx); - goto bail; - } - key = scanstring_str(pystr, idx + 1, encoding, s->strict, &next_idx); - if (key == NULL) - goto bail; - memokey = PyDict_GetItem(s->memo, key); - if (memokey != NULL) { - Py_INCREF(memokey); - Py_DECREF(key); - key = memokey; - } - else { - if (PyDict_SetItem(s->memo, key, key) < 0) - goto bail; - } - idx = next_idx; - - /* skip whitespace between key and : delimiter, read :, skip whitespace */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - if (idx > end_idx || str[idx] != ':') { - raise_errmsg(ERR_OBJECT_PROPERTY_DELIMITER, pystr, idx); - goto bail; - } - idx++; - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* read any JSON data type */ - val = scan_once_str(s, pystr, idx, &next_idx); - if (val == NULL) - goto bail; - - if (has_pairs_hook) { - item = PyTuple_Pack(2, key, val); - if (item == NULL) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - if (PyList_Append(pairs, item) == -1) { - Py_DECREF(item); - goto bail; - } - Py_DECREF(item); - } - else { - if (PyDict_SetItem(rval, key, val) < 0) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - } - idx = next_idx; - - /* skip whitespace before } or , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* bail if the object is closed or we didn't get the , delimiter */ - did_parse = 1; - if (idx > end_idx) break; - if (str[idx] == '}') { - break; - } - else if (str[idx] != ',') { - raise_errmsg(ERR_OBJECT_DELIMITER, pystr, idx); - goto bail; - } - idx++; - - /* skip whitespace after , delimiter */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - trailing_delimiter = 1; - } - if (trailing_delimiter) { - raise_errmsg(ERR_OBJECT_PROPERTY, pystr, idx); - goto bail; - } - } - /* verify that idx < end_idx, str[idx] should be '}' */ - if (idx > end_idx || str[idx] != '}') { - if (did_parse) { - raise_errmsg(ERR_OBJECT_DELIMITER, pystr, idx); - } else { - raise_errmsg(ERR_OBJECT_PROPERTY_FIRST, pystr, idx); - } - goto bail; - } - - /* if pairs_hook is not None: rval = object_pairs_hook(pairs) */ - if (s->pairs_hook != Py_None) { - val = PyObject_CallOneArg(s->pairs_hook, pairs); - if (val == NULL) - goto bail; - Py_DECREF(pairs); - *next_idx_ptr = idx + 1; - return val; - } - - /* if object_hook is not None: rval = object_hook(rval) */ - if (s->object_hook != Py_None) { - val = PyObject_CallOneArg(s->object_hook, rval); - if (val == NULL) - goto bail; - Py_DECREF(rval); - rval = val; - val = NULL; - } - *next_idx_ptr = idx + 1; - return rval; -bail: - Py_XDECREF(rval); - Py_XDECREF(key); - Py_XDECREF(val); - Py_XDECREF(pairs); - return NULL; -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -_parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON object from PyUnicode pystr. - idx is the index of the first character after the opening curly brace. - *next_idx_ptr is a return-by-reference index to the first character after - the closing curly brace. - - Returns a new PyObject (usually a dict, but object_hook can change that) - */ - void *str = PyUnicode_DATA(pystr); - Py_ssize_t end_idx = PyUnicode_GET_LENGTH(pystr) - 1; - PY2_UNUSED int kind = PyUnicode_KIND(pystr); - PyObject *rval = NULL; - PyObject *pairs = NULL; - PyObject *item; - PyObject *key = NULL; - PyObject *val = NULL; - int has_pairs_hook = (s->pairs_hook != Py_None); - int did_parse = 0; - Py_ssize_t next_idx; - - if (has_pairs_hook) { - pairs = PyList_New(0); - if (pairs == NULL) - return NULL; - } - else { - rval = PyDict_New(); - if (rval == NULL) - return NULL; - } - - /* skip whitespace after { */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - - /* only loop if the object is non-empty */ - if (idx <= end_idx && PyUnicode_READ(kind, str, idx) != '}') { - int trailing_delimiter = 0; - while (idx <= end_idx) { - PyObject *memokey; - trailing_delimiter = 0; - - /* read key */ - if (PyUnicode_READ(kind, str, idx) != '"') { - raise_errmsg(ERR_OBJECT_PROPERTY, pystr, idx); - goto bail; - } - key = scanstring_unicode(pystr, idx + 1, s->strict, &next_idx); - if (key == NULL) - goto bail; - memokey = PyDict_GetItem(s->memo, key); - if (memokey != NULL) { - Py_INCREF(memokey); - Py_DECREF(key); - key = memokey; - } - else { - if (PyDict_SetItem(s->memo, key, key) < 0) - goto bail; - } - idx = next_idx; - - /* skip whitespace between key and : delimiter, read :, skip - whitespace */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - if (idx > end_idx || PyUnicode_READ(kind, str, idx) != ':') { - raise_errmsg(ERR_OBJECT_PROPERTY_DELIMITER, pystr, idx); - goto bail; - } - idx++; - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - - /* read any JSON term */ - val = scan_once_unicode(s, pystr, idx, &next_idx); - if (val == NULL) - goto bail; - - if (has_pairs_hook) { - item = PyTuple_Pack(2, key, val); - if (item == NULL) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - if (PyList_Append(pairs, item) == -1) { - Py_DECREF(item); - goto bail; - } - Py_DECREF(item); - } - else { - if (PyDict_SetItem(rval, key, val) < 0) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - } - idx = next_idx; - - /* skip whitespace before } or , */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - - /* bail if the object is closed or we didn't get the , - delimiter */ - did_parse = 1; - if (idx > end_idx) break; - if (PyUnicode_READ(kind, str, idx) == '}') { - break; - } - else if (PyUnicode_READ(kind, str, idx) != ',') { - raise_errmsg(ERR_OBJECT_DELIMITER, pystr, idx); - goto bail; - } - idx++; - - /* skip whitespace after , delimiter */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - trailing_delimiter = 1; - } - if (trailing_delimiter) { - raise_errmsg(ERR_OBJECT_PROPERTY, pystr, idx); - goto bail; - } - } - - /* verify that idx < end_idx, str[idx] should be '}' */ - if (idx > end_idx || PyUnicode_READ(kind, str, idx) != '}') { - if (did_parse) { - raise_errmsg(ERR_OBJECT_DELIMITER, pystr, idx); - } else { - raise_errmsg(ERR_OBJECT_PROPERTY_FIRST, pystr, idx); - } - goto bail; - } - - /* if pairs_hook is not None: rval = object_pairs_hook(pairs) */ - if (s->pairs_hook != Py_None) { - val = PyObject_CallOneArg(s->pairs_hook, pairs); - if (val == NULL) - goto bail; - Py_DECREF(pairs); - *next_idx_ptr = idx + 1; - return val; - } - - /* if object_hook is not None: rval = object_hook(rval) */ - if (s->object_hook != Py_None) { - val = PyObject_CallOneArg(s->object_hook, rval); - if (val == NULL) - goto bail; - Py_DECREF(rval); - rval = val; - val = NULL; - } - *next_idx_ptr = idx + 1; - return rval; -bail: - Py_XDECREF(rval); - Py_XDECREF(key); - Py_XDECREF(val); - Py_XDECREF(pairs); - return NULL; -} - -#if PY_MAJOR_VERSION < 3 -static PyObject * -_parse_array_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON array from PyString pystr. - idx is the index of the first character after the opening brace. - *next_idx_ptr is a return-by-reference index to the first character after - the closing brace. - - Returns a new PyList - */ - char *str = PyString_AS_STRING(pystr); - Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1; - PyObject *val = NULL; - PyObject *rval = PyList_New(0); - Py_ssize_t next_idx; - if (rval == NULL) - return NULL; - - /* skip whitespace after [ */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* only loop if the array is non-empty */ - if (idx <= end_idx && str[idx] != ']') { - int trailing_delimiter = 0; - while (idx <= end_idx) { - trailing_delimiter = 0; - /* read any JSON term and de-tuplefy the (rval, idx) */ - val = scan_once_str(s, pystr, idx, &next_idx); - if (val == NULL) { - goto bail; - } - - if (PyList_Append(rval, val) == -1) - goto bail; - - Py_CLEAR(val); - idx = next_idx; - - /* skip whitespace between term and , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* bail if the array is closed or we didn't get the , delimiter */ - if (idx > end_idx) break; - if (str[idx] == ']') { - break; - } - else if (str[idx] != ',') { - raise_errmsg(ERR_ARRAY_DELIMITER, pystr, idx); - goto bail; - } - idx++; - - /* skip whitespace after , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - trailing_delimiter = 1; - } - if (trailing_delimiter) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx); - goto bail; - } - } - - /* verify that idx < end_idx, str[idx] should be ']' */ - if (idx > end_idx || str[idx] != ']') { - if (PyList_GET_SIZE(rval)) { - raise_errmsg(ERR_ARRAY_DELIMITER, pystr, idx); - } else { - raise_errmsg(ERR_ARRAY_VALUE_FIRST, pystr, idx); - } - goto bail; - } - *next_idx_ptr = idx + 1; - return rval; -bail: - Py_XDECREF(val); - Py_DECREF(rval); - return NULL; -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -_parse_array_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON array from PyString pystr. - idx is the index of the first character after the opening brace. - *next_idx_ptr is a return-by-reference index to the first character after - the closing brace. - - Returns a new PyList - */ - PY2_UNUSED int kind = PyUnicode_KIND(pystr); - void *str = PyUnicode_DATA(pystr); - Py_ssize_t end_idx = PyUnicode_GET_LENGTH(pystr) - 1; - PyObject *val = NULL; - PyObject *rval = PyList_New(0); - Py_ssize_t next_idx; - if (rval == NULL) - return NULL; - - /* skip whitespace after [ */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - - /* only loop if the array is non-empty */ - if (idx <= end_idx && PyUnicode_READ(kind, str, idx) != ']') { - int trailing_delimiter = 0; - while (idx <= end_idx) { - trailing_delimiter = 0; - /* read any JSON term */ - val = scan_once_unicode(s, pystr, idx, &next_idx); - if (val == NULL) { - goto bail; - } - - if (PyList_Append(rval, val) == -1) - goto bail; - - Py_CLEAR(val); - idx = next_idx; - - /* skip whitespace between term and , */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - - /* bail if the array is closed or we didn't get the , delimiter */ - if (idx > end_idx) break; - if (PyUnicode_READ(kind, str, idx) == ']') { - break; - } - else if (PyUnicode_READ(kind, str, idx) != ',') { - raise_errmsg(ERR_ARRAY_DELIMITER, pystr, idx); - goto bail; - } - idx++; - - /* skip whitespace after , */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - trailing_delimiter = 1; - } - if (trailing_delimiter) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx); - goto bail; - } - } - - /* verify that idx < end_idx, str[idx] should be ']' */ - if (idx > end_idx || PyUnicode_READ(kind, str, idx) != ']') { - if (PyList_GET_SIZE(rval)) { - raise_errmsg(ERR_ARRAY_DELIMITER, pystr, idx); - } else { - raise_errmsg(ERR_ARRAY_VALUE_FIRST, pystr, idx); - } - goto bail; - } - *next_idx_ptr = idx + 1; - return rval; -bail: - Py_XDECREF(val); - Py_DECREF(rval); - return NULL; -} - -static PyObject * -_parse_constant(PyScannerObject *s, PyObject *pystr, PyObject *constant, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON constant from PyString pystr. - constant is the Python string that was found - ("NaN", "Infinity", "-Infinity"). - idx is the index of the first character of the constant - *next_idx_ptr is a return-by-reference index to the first character after - the constant. - - Returns the result of parse_constant - */ - PyObject *rval; - if (s->parse_constant == Py_None) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx); - return NULL; - } - - /* rval = parse_constant(constant) */ - rval = PyObject_CallOneArg(s->parse_constant, constant); - idx += PyString_GET_SIZE(constant); - *next_idx_ptr = idx; - return rval; -} - -#if PY_MAJOR_VERSION < 3 -static PyObject * -_match_number_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON number from PyString pystr. - idx is the index of the first character of the number - *next_idx_ptr is a return-by-reference index to the first character after - the number. - - Returns a new PyObject representation of that number: - PyInt, PyLong, or PyFloat. - May return other types if parse_int or parse_float are set - */ - char *str = PyString_AS_STRING(pystr); - Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1; - Py_ssize_t idx = start; - int is_float = 0; - PyObject *rval; - PyObject *numstr; - - /* read a sign if it's there, make sure it's not the end of the string */ - if (str[idx] == '-') { - if (idx >= end_idx) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, start); - return NULL; - } - idx++; - } - - /* read as many integer digits as we find as long as it doesn't start with 0 */ - if (str[idx] >= '1' && str[idx] <= '9') { - idx++; - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; - } - /* if it starts with 0 we only expect one integer digit */ - else if (str[idx] == '0') { - idx++; - } - /* no integer digits, error */ - else { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, start); - return NULL; - } - - /* if the next char is '.' followed by a digit then read all float digits */ - if (idx < end_idx && str[idx] == '.' && str[idx + 1] >= '0' && str[idx + 1] <= '9') { - is_float = 1; - idx += 2; - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; - } - - /* if the next char is 'e' or 'E' then maybe read the exponent (or backtrack) */ - if (idx < end_idx && (str[idx] == 'e' || str[idx] == 'E')) { - - /* save the index of the 'e' or 'E' just in case we need to backtrack */ - Py_ssize_t e_start = idx; - idx++; - - /* read an exponent sign if present */ - if (idx < end_idx && (str[idx] == '-' || str[idx] == '+')) idx++; - - /* read all digits */ - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; - - /* if we got a digit, then parse as float. if not, backtrack */ - if (str[idx - 1] >= '0' && str[idx - 1] <= '9') { - is_float = 1; - } - else { - idx = e_start; - } - } - - /* copy the section we determined to be a number */ - numstr = PyString_FromStringAndSize(&str[start], idx - start); - if (numstr == NULL) - return NULL; - if (is_float) { - /* parse as a float using a fast path if available, otherwise call user defined method */ - if (s->parse_float != (PyObject *)&PyFloat_Type) { - rval = PyObject_CallOneArg(s->parse_float, numstr); - } - else { - /* rval = PyFloat_FromDouble(PyOS_ascii_atof(PyString_AS_STRING(numstr))); */ - double d = PyOS_string_to_double(PyString_AS_STRING(numstr), - NULL, NULL); - if (d == -1.0 && PyErr_Occurred()) { - Py_DECREF(numstr); - return NULL; - } - rval = PyFloat_FromDouble(d); - } - } - else { - /* parse as an int using a fast path if available, otherwise call user defined method */ - if (s->parse_int != (PyObject *)&PyInt_Type) { - rval = PyObject_CallOneArg(s->parse_int, numstr); - } - else { - rval = PyInt_FromString(PyString_AS_STRING(numstr), NULL, 10); - } - } - Py_DECREF(numstr); - *next_idx_ptr = idx; - return rval; -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -_match_number_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON number from PyUnicode pystr. - idx is the index of the first character of the number - *next_idx_ptr is a return-by-reference index to the first character after - the number. - - Returns a new PyObject representation of that number: - PyInt, PyLong, or PyFloat. - May return other types if parse_int or parse_float are set - */ - PY2_UNUSED int kind = PyUnicode_KIND(pystr); - void *str = PyUnicode_DATA(pystr); - Py_ssize_t end_idx = PyUnicode_GET_LENGTH(pystr) - 1; - Py_ssize_t idx = start; - int is_float = 0; - JSON_UNICHR c; - PyObject *rval; - PyObject *numstr; - - /* read a sign if it's there, make sure it's not the end of the string */ - if (PyUnicode_READ(kind, str, idx) == '-') { - if (idx >= end_idx) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, start); - return NULL; - } - idx++; - } - - /* read as many integer digits as we find as long as it doesn't start with 0 */ - c = PyUnicode_READ(kind, str, idx); - if (c == '0') { - /* if it starts with 0 we only expect one integer digit */ - idx++; - } - else if (IS_DIGIT(c)) { - idx++; - while (idx <= end_idx && IS_DIGIT(PyUnicode_READ(kind, str, idx))) { - idx++; - } - } - else { - /* no integer digits, error */ - raise_errmsg(ERR_EXPECTING_VALUE, pystr, start); - return NULL; - } - - /* if the next char is '.' followed by a digit then read all float digits */ - if (idx < end_idx && - PyUnicode_READ(kind, str, idx) == '.' && - IS_DIGIT(PyUnicode_READ(kind, str, idx + 1))) { - is_float = 1; - idx += 2; - while (idx <= end_idx && IS_DIGIT(PyUnicode_READ(kind, str, idx))) idx++; - } - - /* if the next char is 'e' or 'E' then maybe read the exponent (or backtrack) */ - if (idx < end_idx && - (PyUnicode_READ(kind, str, idx) == 'e' || - PyUnicode_READ(kind, str, idx) == 'E')) { - Py_ssize_t e_start = idx; - idx++; - - /* read an exponent sign if present */ - if (idx < end_idx && - (PyUnicode_READ(kind, str, idx) == '-' || - PyUnicode_READ(kind, str, idx) == '+')) idx++; - - /* read all digits */ - while (idx <= end_idx && IS_DIGIT(PyUnicode_READ(kind, str, idx))) idx++; - - /* if we got a digit, then parse as float. if not, backtrack */ - if (IS_DIGIT(PyUnicode_READ(kind, str, idx - 1))) { - is_float = 1; - } - else { - idx = e_start; - } - } - - /* copy the section we determined to be a number */ -#if PY_MAJOR_VERSION >= 3 - numstr = PyUnicode_Substring(pystr, start, idx); -#else - numstr = PyUnicode_FromUnicode(&((Py_UNICODE *)str)[start], idx - start); -#endif - if (numstr == NULL) - return NULL; - if (is_float) { - /* parse as a float using a fast path if available, otherwise call user defined method */ - if (s->parse_float != (PyObject *)&PyFloat_Type) { - rval = PyObject_CallOneArg(s->parse_float, numstr); - } - else { -#if PY_MAJOR_VERSION >= 3 - rval = PyFloat_FromString(numstr); -#else - rval = PyFloat_FromString(numstr, NULL); -#endif - } - } - else { - /* no fast path for unicode -> int, just call */ - rval = PyObject_CallOneArg(s->parse_int, numstr); - } - Py_DECREF(numstr); - *next_idx_ptr = idx; - return rval; -} - -#if PY_MAJOR_VERSION < 3 -static PyObject * -scan_once_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read one JSON term (of any kind) from PyString pystr. - idx is the index of the first character of the term - *next_idx_ptr is a return-by-reference index to the first character after - the number. - - Returns a new PyObject representation of the term. - */ - char *str = PyString_AS_STRING(pystr); - Py_ssize_t length = PyString_GET_SIZE(pystr); - PyObject *rval = NULL; - int fallthrough = 0; - if (idx < 0 || idx >= length) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx); - return NULL; - } - switch (str[idx]) { - case '"': - /* string */ - rval = scanstring_str(pystr, idx + 1, - PyString_AS_STRING(s->encoding), - s->strict, - next_idx_ptr); - break; - case '{': - /* object */ - if (Py_EnterRecursiveCall(" while decoding a JSON object " - "from a string")) - return NULL; - rval = _parse_object_str(s, pystr, idx + 1, next_idx_ptr); - Py_LeaveRecursiveCall(); - break; - case '[': - /* array */ - if (Py_EnterRecursiveCall(" while decoding a JSON array " - "from a string")) - return NULL; - rval = _parse_array_str(s, pystr, idx + 1, next_idx_ptr); - Py_LeaveRecursiveCall(); - break; - case 'n': - /* null */ - if ((idx + 3 < length) && str[idx + 1] == 'u' && str[idx + 2] == 'l' && str[idx + 3] == 'l') { - Py_INCREF(Py_None); - *next_idx_ptr = idx + 4; - rval = Py_None; - } - else - fallthrough = 1; - break; - case 't': - /* true */ - if ((idx + 3 < length) && str[idx + 1] == 'r' && str[idx + 2] == 'u' && str[idx + 3] == 'e') { - Py_INCREF(Py_True); - *next_idx_ptr = idx + 4; - rval = Py_True; - } - else - fallthrough = 1; - break; - case 'f': - /* false */ - if ((idx + 4 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'l' && str[idx + 3] == 's' && str[idx + 4] == 'e') { - Py_INCREF(Py_False); - *next_idx_ptr = idx + 5; - rval = Py_False; - } - else - fallthrough = 1; - break; - case 'N': - /* NaN */ - if ((idx + 2 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'N') { - rval = _parse_constant(s, pystr, JSON_NaN, idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - case 'I': - /* Infinity */ - if ((idx + 7 < length) && str[idx + 1] == 'n' && str[idx + 2] == 'f' && str[idx + 3] == 'i' && str[idx + 4] == 'n' && str[idx + 5] == 'i' && str[idx + 6] == 't' && str[idx + 7] == 'y') { - rval = _parse_constant(s, pystr, JSON_Infinity, idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - case '-': - /* -Infinity */ - if ((idx + 8 < length) && str[idx + 1] == 'I' && str[idx + 2] == 'n' && str[idx + 3] == 'f' && str[idx + 4] == 'i' && str[idx + 5] == 'n' && str[idx + 6] == 'i' && str[idx + 7] == 't' && str[idx + 8] == 'y') { - rval = _parse_constant(s, pystr, JSON_NegInfinity, idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - default: - fallthrough = 1; - } - /* Didn't find a string, object, array, or named constant. Look for a number. */ - if (fallthrough) - rval = _match_number_str(s, pystr, idx, next_idx_ptr); - return rval; -} -#endif /* PY_MAJOR_VERSION < 3 */ - - -static PyObject * -scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read one JSON term (of any kind) from PyUnicode pystr. - idx is the index of the first character of the term - *next_idx_ptr is a return-by-reference index to the first character after - the number. - - Returns a new PyObject representation of the term. - */ - PY2_UNUSED int kind = PyUnicode_KIND(pystr); - void *str = PyUnicode_DATA(pystr); - Py_ssize_t length = PyUnicode_GET_LENGTH(pystr); - PyObject *rval = NULL; - int fallthrough = 0; - if (idx < 0 || idx >= length) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx); - return NULL; - } - switch (PyUnicode_READ(kind, str, idx)) { - case '"': - /* string */ - rval = scanstring_unicode(pystr, idx + 1, - s->strict, - next_idx_ptr); - break; - case '{': - /* object */ - if (Py_EnterRecursiveCall(" while decoding a JSON object " - "from a unicode string")) - return NULL; - rval = _parse_object_unicode(s, pystr, idx + 1, next_idx_ptr); - Py_LeaveRecursiveCall(); - break; - case '[': - /* array */ - if (Py_EnterRecursiveCall(" while decoding a JSON array " - "from a unicode string")) - return NULL; - rval = _parse_array_unicode(s, pystr, idx + 1, next_idx_ptr); - Py_LeaveRecursiveCall(); - break; - case 'n': - /* null */ - if ((idx + 3 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'u' && - PyUnicode_READ(kind, str, idx + 2) == 'l' && - PyUnicode_READ(kind, str, idx + 3) == 'l') { - Py_INCREF(Py_None); - *next_idx_ptr = idx + 4; - rval = Py_None; - } - else - fallthrough = 1; - break; - case 't': - /* true */ - if ((idx + 3 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'r' && - PyUnicode_READ(kind, str, idx + 2) == 'u' && - PyUnicode_READ(kind, str, idx + 3) == 'e') { - Py_INCREF(Py_True); - *next_idx_ptr = idx + 4; - rval = Py_True; - } - else - fallthrough = 1; - break; - case 'f': - /* false */ - if ((idx + 4 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'a' && - PyUnicode_READ(kind, str, idx + 2) == 'l' && - PyUnicode_READ(kind, str, idx + 3) == 's' && - PyUnicode_READ(kind, str, idx + 4) == 'e') { - Py_INCREF(Py_False); - *next_idx_ptr = idx + 5; - rval = Py_False; - } - else - fallthrough = 1; - break; - case 'N': - /* NaN */ - if ((idx + 2 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'a' && - PyUnicode_READ(kind, str, idx + 2) == 'N') { - rval = _parse_constant(s, pystr, JSON_NaN, idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - case 'I': - /* Infinity */ - if ((idx + 7 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'n' && - PyUnicode_READ(kind, str, idx + 2) == 'f' && - PyUnicode_READ(kind, str, idx + 3) == 'i' && - PyUnicode_READ(kind, str, idx + 4) == 'n' && - PyUnicode_READ(kind, str, idx + 5) == 'i' && - PyUnicode_READ(kind, str, idx + 6) == 't' && - PyUnicode_READ(kind, str, idx + 7) == 'y') { - rval = _parse_constant(s, pystr, JSON_Infinity, idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - case '-': - /* -Infinity */ - if ((idx + 8 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'I' && - PyUnicode_READ(kind, str, idx + 2) == 'n' && - PyUnicode_READ(kind, str, idx + 3) == 'f' && - PyUnicode_READ(kind, str, idx + 4) == 'i' && - PyUnicode_READ(kind, str, idx + 5) == 'n' && - PyUnicode_READ(kind, str, idx + 6) == 'i' && - PyUnicode_READ(kind, str, idx + 7) == 't' && - PyUnicode_READ(kind, str, idx + 8) == 'y') { - rval = _parse_constant(s, pystr, JSON_NegInfinity, idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - default: - fallthrough = 1; - } - /* Didn't find a string, object, array, or named constant. Look for a number. */ - if (fallthrough) - rval = _match_number_unicode(s, pystr, idx, next_idx_ptr); - return rval; -} - -static PyObject * -scanner_call(PyObject *self, PyObject *args, PyObject *kwds) -{ - /* Python callable interface to scan_once_{str,unicode} */ - PyObject *pystr; - PyObject *rval; - Py_ssize_t idx; - Py_ssize_t next_idx = -1; - static char *kwlist[] = {"string", "idx", NULL}; - PyScannerObject *s; - assert(PyScanner_Check(self)); - s = (PyScannerObject *)self; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&:scan_once", kwlist, &pystr, _convertPyInt_AsSsize_t, &idx)) - return NULL; - - if (PyUnicode_Check(pystr)) { - if (PyUnicode_READY(pystr)) - return NULL; - rval = scan_once_unicode(s, pystr, idx, &next_idx); - } -#if PY_MAJOR_VERSION < 3 - else if (PyString_Check(pystr)) { - rval = scan_once_str(s, pystr, idx, &next_idx); - } -#endif /* PY_MAJOR_VERSION < 3 */ - else { - PyErr_Format(PyExc_TypeError, - "first argument must be a string, not %.80s", - Py_TYPE(pystr)->tp_name); - return NULL; - } - PyDict_Clear(s->memo); - return _build_rval_index_tuple(rval, next_idx); -} - -static PyObject * -JSON_ParseEncoding(PyObject *encoding) -{ - if (encoding == Py_None) - return JSON_InternFromString(DEFAULT_ENCODING); -#if PY_MAJOR_VERSION >= 3 - if (PyUnicode_Check(encoding)) { - if (PyUnicode_AsUTF8(encoding) == NULL) { - return NULL; - } - Py_INCREF(encoding); - return encoding; - } -#else /* PY_MAJOR_VERSION >= 3 */ - if (PyString_Check(encoding)) { - Py_INCREF(encoding); - return encoding; - } - if (PyUnicode_Check(encoding)) - return PyUnicode_AsEncodedString(encoding, NULL, NULL); -#endif /* PY_MAJOR_VERSION >= 3 */ - PyErr_SetString(PyExc_TypeError, "encoding must be a string"); - return NULL; -} - -static PyObject * -scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - /* Initialize Scanner object */ - PyObject *ctx; - static char *kwlist[] = {"context", NULL}; - PyScannerObject *s; - PyObject *encoding; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:make_scanner", kwlist, &ctx)) - return NULL; - - s = (PyScannerObject *)type->tp_alloc(type, 0); - if (s == NULL) - return NULL; - - if (s->memo == NULL) { - s->memo = PyDict_New(); - if (s->memo == NULL) - goto bail; - } - - encoding = PyObject_GetAttrString(ctx, "encoding"); - if (encoding == NULL) - goto bail; - s->encoding = JSON_ParseEncoding(encoding); - Py_XDECREF(encoding); - if (s->encoding == NULL) - goto bail; - - /* All of these will fail "gracefully" so we don't need to verify them */ - s->strict_bool = PyObject_GetAttrString(ctx, "strict"); - if (s->strict_bool == NULL) - goto bail; - s->strict = PyObject_IsTrue(s->strict_bool); - if (s->strict < 0) - goto bail; - s->object_hook = PyObject_GetAttrString(ctx, "object_hook"); - if (s->object_hook == NULL) - goto bail; - s->pairs_hook = PyObject_GetAttrString(ctx, "object_pairs_hook"); - if (s->pairs_hook == NULL) - goto bail; - s->parse_float = PyObject_GetAttrString(ctx, "parse_float"); - if (s->parse_float == NULL) - goto bail; - s->parse_int = PyObject_GetAttrString(ctx, "parse_int"); - if (s->parse_int == NULL) - goto bail; - s->parse_constant = PyObject_GetAttrString(ctx, "parse_constant"); - if (s->parse_constant == NULL) - goto bail; - - return (PyObject *)s; - -bail: - Py_DECREF(s); - return NULL; -} - -PyDoc_STRVAR(scanner_doc, "JSON scanner object"); - -static -PyTypeObject PyScannerType = { - PyVarObject_HEAD_INIT(NULL, 0) - "simplejson._speedups.Scanner", /* tp_name */ - sizeof(PyScannerObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - scanner_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - scanner_call, /* tp_call */ - 0, /* tp_str */ - 0,/* PyObject_GenericGetAttr, */ /* tp_getattro */ - 0,/* PyObject_GenericSetAttr, */ /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - scanner_doc, /* tp_doc */ - scanner_traverse, /* tp_traverse */ - scanner_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - scanner_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0,/* PyType_GenericAlloc, */ /* tp_alloc */ - scanner_new, /* tp_new */ - 0,/* PyObject_GC_Del, */ /* tp_free */ -}; - -static PyObject * -encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = { - "markers", - "default", - "encoder", - "indent", - "key_separator", - "item_separator", - "sort_keys", - "skipkeys", - "allow_nan", - "key_memo", - "use_decimal", - "namedtuple_as_object", - "tuple_as_array", - "int_as_string_bitcount", - "item_sort_key", - "encoding", - "for_json", - "ignore_nan", - "Decimal", - "iterable_as_array", - NULL}; - - PyEncoderObject *s; - PyObject *markers, *defaultfn, *encoder, *indent, *key_separator; - PyObject *item_separator, *sort_keys, *skipkeys, *allow_nan, *key_memo; - PyObject *use_decimal, *namedtuple_as_object, *tuple_as_array, *iterable_as_array; - PyObject *int_as_string_bitcount, *item_sort_key, *encoding, *for_json; - PyObject *ignore_nan, *Decimal; - int is_true; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOOOOOOOOOOOOOOOOOOO:make_encoder", kwlist, - &markers, &defaultfn, &encoder, &indent, &key_separator, &item_separator, - &sort_keys, &skipkeys, &allow_nan, &key_memo, &use_decimal, - &namedtuple_as_object, &tuple_as_array, - &int_as_string_bitcount, &item_sort_key, &encoding, &for_json, - &ignore_nan, &Decimal, &iterable_as_array)) - return NULL; - - s = (PyEncoderObject *)type->tp_alloc(type, 0); - if (s == NULL) - return NULL; - - Py_INCREF(markers); - s->markers = markers; - Py_INCREF(defaultfn); - s->defaultfn = defaultfn; - Py_INCREF(encoder); - s->encoder = encoder; -#if PY_MAJOR_VERSION >= 3 - if (encoding == Py_None) { - s->encoding = NULL; - } - else -#endif /* PY_MAJOR_VERSION >= 3 */ - { - s->encoding = JSON_ParseEncoding(encoding); - if (s->encoding == NULL) - goto bail; - } - Py_INCREF(indent); - s->indent = indent; - Py_INCREF(key_separator); - s->key_separator = key_separator; - Py_INCREF(item_separator); - s->item_separator = item_separator; - Py_INCREF(skipkeys); - s->skipkeys_bool = skipkeys; - s->skipkeys = PyObject_IsTrue(skipkeys); - if (s->skipkeys < 0) - goto bail; - Py_INCREF(key_memo); - s->key_memo = key_memo; - s->fast_encode = (PyCFunction_Check(s->encoder) && PyCFunction_GetFunction(s->encoder) == (PyCFunction)py_encode_basestring_ascii); - is_true = PyObject_IsTrue(ignore_nan); - if (is_true < 0) - goto bail; - s->allow_or_ignore_nan = is_true ? JSON_IGNORE_NAN : 0; - is_true = PyObject_IsTrue(allow_nan); - if (is_true < 0) - goto bail; - s->allow_or_ignore_nan |= is_true ? JSON_ALLOW_NAN : 0; - s->use_decimal = PyObject_IsTrue(use_decimal); - if (s->use_decimal < 0) - goto bail; - s->namedtuple_as_object = PyObject_IsTrue(namedtuple_as_object); - if (s->namedtuple_as_object < 0) - goto bail; - s->tuple_as_array = PyObject_IsTrue(tuple_as_array); - if (s->tuple_as_array < 0) - goto bail; - s->iterable_as_array = PyObject_IsTrue(iterable_as_array); - if (s->iterable_as_array < 0) - goto bail; - if (PyInt_Check(int_as_string_bitcount) || PyLong_Check(int_as_string_bitcount)) { - static const unsigned long long_long_bitsize = SIZEOF_LONG_LONG * 8; - long int_as_string_bitcount_val = PyLong_AsLong(int_as_string_bitcount); - if (int_as_string_bitcount_val > 0 && int_as_string_bitcount_val < (long)long_long_bitsize) { - s->max_long_size = PyLong_FromUnsignedLongLong(1ULL << (int)int_as_string_bitcount_val); - s->min_long_size = PyLong_FromLongLong(-1LL << (int)int_as_string_bitcount_val); - if (s->min_long_size == NULL || s->max_long_size == NULL) { - goto bail; - } - } - else { - PyErr_Format(PyExc_TypeError, - "int_as_string_bitcount (%ld) must be greater than 0 and less than the number of bits of a `long long` type (%lu bits)", - int_as_string_bitcount_val, long_long_bitsize); - goto bail; - } - } - else if (int_as_string_bitcount == Py_None) { - Py_INCREF(Py_None); - s->max_long_size = Py_None; - Py_INCREF(Py_None); - s->min_long_size = Py_None; - } - else { - PyErr_SetString(PyExc_TypeError, "int_as_string_bitcount must be None or an integer"); - goto bail; - } - if (item_sort_key != Py_None) { - if (!PyCallable_Check(item_sort_key)) { - PyErr_SetString(PyExc_TypeError, "item_sort_key must be None or callable"); - goto bail; - } - } - else { - is_true = PyObject_IsTrue(sort_keys); - if (is_true < 0) - goto bail; - if (is_true) { - static PyObject *itemgetter0 = NULL; - if (!itemgetter0) { - PyObject *operator = PyImport_ImportModule("operator"); - if (!operator) - goto bail; - itemgetter0 = PyObject_CallMethod(operator, "itemgetter", "i", 0); - Py_DECREF(operator); - } - item_sort_key = itemgetter0; - if (!item_sort_key) - goto bail; - } - } - if (item_sort_key == Py_None) { - Py_INCREF(Py_None); - s->item_sort_kw = Py_None; - } - else { - s->item_sort_kw = PyDict_New(); - if (s->item_sort_kw == NULL) - goto bail; - if (PyDict_SetItemString(s->item_sort_kw, "key", item_sort_key)) - goto bail; - } - Py_INCREF(sort_keys); - s->sort_keys = sort_keys; - Py_INCREF(item_sort_key); - s->item_sort_key = item_sort_key; - Py_INCREF(Decimal); - s->Decimal = Decimal; - s->for_json = PyObject_IsTrue(for_json); - if (s->for_json < 0) - goto bail; - - return (PyObject *)s; - -bail: - Py_DECREF(s); - return NULL; -} - -static PyObject * -encoder_call(PyObject *self, PyObject *args, PyObject *kwds) -{ - /* Python callable interface to encode_listencode_obj */ - static char *kwlist[] = {"obj", "_current_indent_level", NULL}; - PyObject *obj; - Py_ssize_t indent_level; - PyEncoderObject *s; - JSON_Accu rval; - assert(PyEncoder_Check(self)); - s = (PyEncoderObject *)self; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&:_iterencode", kwlist, - &obj, _convertPyInt_AsSsize_t, &indent_level)) - return NULL; - if (JSON_Accu_Init(&rval)) - return NULL; - if (encoder_listencode_obj(s, &rval, obj, indent_level)) { - JSON_Accu_Destroy(&rval); - return NULL; - } - return JSON_Accu_FinishAsList(&rval); -} - -static PyObject * -_encoded_const(PyObject *obj) -{ - /* Return the JSON string representation of None, True, False */ - if (obj == Py_None) { - static PyObject *s_null = NULL; - if (s_null == NULL) { - s_null = JSON_InternFromString("null"); - } - Py_INCREF(s_null); - return s_null; - } - else if (obj == Py_True) { - static PyObject *s_true = NULL; - if (s_true == NULL) { - s_true = JSON_InternFromString("true"); - } - Py_INCREF(s_true); - return s_true; - } - else if (obj == Py_False) { - static PyObject *s_false = NULL; - if (s_false == NULL) { - s_false = JSON_InternFromString("false"); - } - Py_INCREF(s_false); - return s_false; - } - else { - PyErr_SetString(PyExc_ValueError, "not a const"); - return NULL; - } -} - -static PyObject * -encoder_encode_float(PyEncoderObject *s, PyObject *obj) -{ - /* Return the JSON representation of a PyFloat */ - double i = PyFloat_AS_DOUBLE(obj); - if (!Py_IS_FINITE(i)) { - if (!s->allow_or_ignore_nan) { - PyErr_SetString(PyExc_ValueError, "Out of range float values are not JSON compliant"); - return NULL; - } - if (s->allow_or_ignore_nan & JSON_IGNORE_NAN) { - return _encoded_const(Py_None); - } - /* JSON_ALLOW_NAN is set */ - else if (i > 0) { - Py_INCREF(JSON_Infinity); - return JSON_Infinity; - } - else if (i < 0) { - Py_INCREF(JSON_NegInfinity); - return JSON_NegInfinity; - } - else { - Py_INCREF(JSON_NaN); - return JSON_NaN; - } - } - /* Use a better float format here? */ - if (PyFloat_CheckExact(obj)) { - return PyObject_Repr(obj); - } - else { - /* See #118, do not trust custom str/repr */ - PyObject *res; - PyObject *tmp = PyObject_CallOneArg((PyObject *)&PyFloat_Type, obj); - if (tmp == NULL) { - return NULL; - } - res = PyObject_Repr(tmp); - Py_DECREF(tmp); - return res; - } -} - -static PyObject * -encoder_encode_string(PyEncoderObject *s, PyObject *obj) -{ - /* Return the JSON representation of a string */ - PyObject *encoded; - - if (s->fast_encode) { - return py_encode_basestring_ascii(NULL, obj); - } - encoded = PyObject_CallOneArg(s->encoder, obj); - if (encoded != NULL && -#if PY_MAJOR_VERSION < 3 - !PyString_Check(encoded) && -#endif /* PY_MAJOR_VERSION < 3 */ - !PyUnicode_Check(encoded)) - { - PyErr_Format(PyExc_TypeError, - "encoder() must return a string, not %.80s", - Py_TYPE(encoded)->tp_name); - Py_DECREF(encoded); - return NULL; - } - return encoded; -} - -static int -_steal_accumulate(JSON_Accu *accu, PyObject *stolen) -{ - /* Append stolen and then decrement its reference count */ - int rval = JSON_Accu_Accumulate(accu, stolen); - Py_DECREF(stolen); - return rval; -} - -static int -encoder_listencode_obj(PyEncoderObject *s, JSON_Accu *rval, PyObject *obj, Py_ssize_t indent_level) -{ - /* Encode Python object obj to a JSON term, rval is a PyList */ - int rv = -1; - do { - PyObject *newobj; - if (obj == Py_None || obj == Py_True || obj == Py_False) { - PyObject *cstr = _encoded_const(obj); - if (cstr != NULL) - rv = _steal_accumulate(rval, cstr); - } - else if ((PyBytes_Check(obj) && s->encoding != NULL) || - PyUnicode_Check(obj)) - { - PyObject *encoded = encoder_encode_string(s, obj); - if (encoded != NULL) - rv = _steal_accumulate(rval, encoded); - } - else if (PyInt_Check(obj) || PyLong_Check(obj)) { - PyObject *encoded; - if (PyInt_CheckExact(obj) || PyLong_CheckExact(obj)) { - encoded = PyObject_Str(obj); - } - else { - /* See #118, do not trust custom str/repr */ - PyObject *tmp = PyObject_CallOneArg((PyObject *)&PyLong_Type, obj); - if (tmp == NULL) { - encoded = NULL; - } - else { - encoded = PyObject_Str(tmp); - Py_DECREF(tmp); - } - } - if (encoded != NULL) { - encoded = maybe_quote_bigint(s, encoded, obj); - if (encoded == NULL) - break; - rv = _steal_accumulate(rval, encoded); - } - } - else if (PyFloat_Check(obj)) { - PyObject *encoded = encoder_encode_float(s, obj); - if (encoded != NULL) - rv = _steal_accumulate(rval, encoded); - } - else if (s->for_json && _call_json_method(obj, FOR_JSON_METHOD_NAME, &newobj)) { - if (newobj == NULL) { - return -1; - } - if (Py_EnterRecursiveCall(" while encoding a JSON object")) { - Py_DECREF(newobj); - return rv; - } - rv = encoder_listencode_obj(s, rval, newobj, indent_level); - Py_DECREF(newobj); - Py_LeaveRecursiveCall(); - } - else if (s->namedtuple_as_object && _call_json_method(obj, ASDICT_METHOD_NAME, &newobj)) { - if (newobj == NULL) { - return -1; - } - if (Py_EnterRecursiveCall(" while encoding a JSON object")) { - Py_DECREF(newobj); - return rv; - } - if (PyDict_Check(newobj)) { - rv = encoder_listencode_dict(s, rval, newobj, indent_level); - } else { - PyErr_Format( - PyExc_TypeError, - "_asdict() must return a dict, not %.80s", - Py_TYPE(newobj)->tp_name - ); - rv = -1; - } - Py_DECREF(newobj); - Py_LeaveRecursiveCall(); - } - else if (PyList_Check(obj) || (s->tuple_as_array && PyTuple_Check(obj))) { - if (Py_EnterRecursiveCall(" while encoding a JSON object")) - return rv; - rv = encoder_listencode_list(s, rval, obj, indent_level); - Py_LeaveRecursiveCall(); - } - else if (PyDict_Check(obj)) { - if (Py_EnterRecursiveCall(" while encoding a JSON object")) - return rv; - rv = encoder_listencode_dict(s, rval, obj, indent_level); - Py_LeaveRecursiveCall(); - } - else if (s->use_decimal && PyObject_TypeCheck(obj, (PyTypeObject *)s->Decimal)) { - PyObject *encoded = PyObject_Str(obj); - if (encoded != NULL) - rv = _steal_accumulate(rval, encoded); - } - else if (is_raw_json(obj)) - { - PyObject *encoded = PyObject_GetAttrString(obj, "encoded_json"); - if (encoded != NULL) - rv = _steal_accumulate(rval, encoded); - } - else { - PyObject *ident = NULL; - PyObject *newobj; - if (s->iterable_as_array) { - newobj = PyObject_GetIter(obj); - if (newobj == NULL) - PyErr_Clear(); - else { - rv = encoder_listencode_list(s, rval, newobj, indent_level); - Py_DECREF(newobj); - break; - } - } - if (s->markers != Py_None) { - int has_key; - ident = PyLong_FromVoidPtr(obj); - if (ident == NULL) - break; - has_key = PyDict_Contains(s->markers, ident); - if (has_key) { - if (has_key != -1) - PyErr_SetString(PyExc_ValueError, "Circular reference detected"); - Py_DECREF(ident); - break; - } - if (PyDict_SetItem(s->markers, ident, obj)) { - Py_DECREF(ident); - break; - } - } - if (Py_EnterRecursiveCall(" while encoding a JSON object")) - return rv; - newobj = PyObject_CallOneArg(s->defaultfn, obj); - if (newobj == NULL) { - Py_XDECREF(ident); - Py_LeaveRecursiveCall(); - break; - } - rv = encoder_listencode_obj(s, rval, newobj, indent_level); - Py_LeaveRecursiveCall(); - Py_DECREF(newobj); - if (rv) { - Py_XDECREF(ident); - rv = -1; - } - else if (ident != NULL) { - if (PyDict_DelItem(s->markers, ident)) { - Py_XDECREF(ident); - rv = -1; - } - Py_XDECREF(ident); - } - } - } while (0); - return rv; -} - -static int -encoder_listencode_dict(PyEncoderObject *s, JSON_Accu *rval, PyObject *dct, Py_ssize_t indent_level) -{ - /* Encode Python dict dct a JSON term */ - static PyObject *open_dict = NULL; - static PyObject *close_dict = NULL; - static PyObject *empty_dict = NULL; - PyObject *kstr = NULL; - PyObject *ident = NULL; - PyObject *iter = NULL; - PyObject *item = NULL; - PyObject *items = NULL; - PyObject *encoded = NULL; - Py_ssize_t idx; - - if (open_dict == NULL || close_dict == NULL || empty_dict == NULL) { - open_dict = JSON_InternFromString("{"); - close_dict = JSON_InternFromString("}"); - empty_dict = JSON_InternFromString("{}"); - if (open_dict == NULL || close_dict == NULL || empty_dict == NULL) - return -1; - } - if (PyDict_Size(dct) == 0) - return JSON_Accu_Accumulate(rval, empty_dict); - - if (s->markers != Py_None) { - int has_key; - ident = PyLong_FromVoidPtr(dct); - if (ident == NULL) - goto bail; - has_key = PyDict_Contains(s->markers, ident); - if (has_key) { - if (has_key != -1) - PyErr_SetString(PyExc_ValueError, "Circular reference detected"); - goto bail; - } - if (PyDict_SetItem(s->markers, ident, dct)) { - goto bail; - } - } - - if (JSON_Accu_Accumulate(rval, open_dict)) - goto bail; - - if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level += 1; - /* - newline_indent = '\n' + (_indent * _current_indent_level) - separator = _item_separator + newline_indent - buf += newline_indent - */ - } - - iter = encoder_dict_iteritems(s, dct); - if (iter == NULL) - goto bail; - - idx = 0; - while ((item = PyIter_Next(iter))) { - PyObject *encoded, *key, *value; - if (!PyTuple_Check(item) || Py_SIZE(item) != 2) { - PyErr_SetString(PyExc_ValueError, "items must return 2-tuples"); - goto bail; - } - key = PyTuple_GET_ITEM(item, 0); - if (key == NULL) - goto bail; - value = PyTuple_GET_ITEM(item, 1); - if (value == NULL) - goto bail; - - kstr = encoder_stringify_key(s, key); - if (kstr == NULL) - goto bail; - else if (kstr == Py_None) { - /* skipkeys */ - Py_DECREF(item); - Py_DECREF(kstr); - continue; - } - if (idx) { - if (JSON_Accu_Accumulate(rval, s->item_separator)) - goto bail; - } - /* - * Only cache the encoding of string keys. False and True are - * indistinguishable from 0 and 1 in a dictionary lookup and there - * may be other quirks with user defined subclasses. - */ - encoded = PyDict_GetItem(s->key_memo, kstr); - if (encoded != NULL) { - Py_INCREF(encoded); - Py_CLEAR(kstr); - } else { - encoded = encoder_encode_string(s, kstr); - Py_CLEAR(kstr); - if (encoded == NULL) - goto bail; - if (PyDict_SetItem(s->key_memo, key, encoded)) - goto bail; - } - if (JSON_Accu_Accumulate(rval, encoded)) { - goto bail; - } - Py_CLEAR(encoded); - if (JSON_Accu_Accumulate(rval, s->key_separator)) - goto bail; - if (encoder_listencode_obj(s, rval, value, indent_level)) - goto bail; - Py_CLEAR(item); - idx += 1; - } - Py_CLEAR(iter); - if (PyErr_Occurred()) - goto bail; - if (ident != NULL) { - if (PyDict_DelItem(s->markers, ident)) - goto bail; - Py_CLEAR(ident); - } - if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level -= 1; - /* - yield '\n' + (_indent * _current_indent_level) - */ - } - if (JSON_Accu_Accumulate(rval, close_dict)) - goto bail; - return 0; - -bail: - Py_XDECREF(encoded); - Py_XDECREF(items); - Py_XDECREF(item); - Py_XDECREF(iter); - Py_XDECREF(kstr); - Py_XDECREF(ident); - return -1; -} - - -static int -encoder_listencode_list(PyEncoderObject *s, JSON_Accu *rval, PyObject *seq, Py_ssize_t indent_level) -{ - /* Encode Python list seq to a JSON term */ - static PyObject *open_array = NULL; - static PyObject *close_array = NULL; - static PyObject *empty_array = NULL; - PyObject *ident = NULL; - PyObject *iter = NULL; - PyObject *obj = NULL; - int is_true; - int i = 0; - - if (open_array == NULL || close_array == NULL || empty_array == NULL) { - open_array = JSON_InternFromString("["); - close_array = JSON_InternFromString("]"); - empty_array = JSON_InternFromString("[]"); - if (open_array == NULL || close_array == NULL || empty_array == NULL) - return -1; - } - ident = NULL; - is_true = PyObject_IsTrue(seq); - if (is_true == -1) - return -1; - else if (is_true == 0) - return JSON_Accu_Accumulate(rval, empty_array); - - if (s->markers != Py_None) { - int has_key; - ident = PyLong_FromVoidPtr(seq); - if (ident == NULL) - goto bail; - has_key = PyDict_Contains(s->markers, ident); - if (has_key) { - if (has_key != -1) - PyErr_SetString(PyExc_ValueError, "Circular reference detected"); - goto bail; - } - if (PyDict_SetItem(s->markers, ident, seq)) { - goto bail; - } - } - - iter = PyObject_GetIter(seq); - if (iter == NULL) - goto bail; - - if (JSON_Accu_Accumulate(rval, open_array)) - goto bail; - if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level += 1; - /* - newline_indent = '\n' + (_indent * _current_indent_level) - separator = _item_separator + newline_indent - buf += newline_indent - */ - } - while ((obj = PyIter_Next(iter))) { - if (i) { - if (JSON_Accu_Accumulate(rval, s->item_separator)) - goto bail; - } - if (encoder_listencode_obj(s, rval, obj, indent_level)) - goto bail; - i++; - Py_CLEAR(obj); - } - Py_CLEAR(iter); - if (PyErr_Occurred()) - goto bail; - if (ident != NULL) { - if (PyDict_DelItem(s->markers, ident)) - goto bail; - Py_CLEAR(ident); - } - if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level -= 1; - /* - yield '\n' + (_indent * _current_indent_level) - */ - } - if (JSON_Accu_Accumulate(rval, close_array)) - goto bail; - return 0; - -bail: - Py_XDECREF(obj); - Py_XDECREF(iter); - Py_XDECREF(ident); - return -1; -} - -static void -encoder_dealloc(PyObject *self) -{ - /* bpo-31095: UnTrack is needed before calling any callbacks */ - PyObject_GC_UnTrack(self); - encoder_clear(self); - Py_TYPE(self)->tp_free(self); -} - -static int -encoder_traverse(PyObject *self, visitproc visit, void *arg) -{ - PyEncoderObject *s; - assert(PyEncoder_Check(self)); - s = (PyEncoderObject *)self; - Py_VISIT(s->markers); - Py_VISIT(s->defaultfn); - Py_VISIT(s->encoder); - Py_VISIT(s->encoding); - Py_VISIT(s->indent); - Py_VISIT(s->key_separator); - Py_VISIT(s->item_separator); - Py_VISIT(s->key_memo); - Py_VISIT(s->sort_keys); - Py_VISIT(s->item_sort_kw); - Py_VISIT(s->item_sort_key); - Py_VISIT(s->max_long_size); - Py_VISIT(s->min_long_size); - Py_VISIT(s->Decimal); - return 0; -} - -static int -encoder_clear(PyObject *self) -{ - /* Deallocate Encoder */ - PyEncoderObject *s; - assert(PyEncoder_Check(self)); - s = (PyEncoderObject *)self; - Py_CLEAR(s->markers); - Py_CLEAR(s->defaultfn); - Py_CLEAR(s->encoder); - Py_CLEAR(s->encoding); - Py_CLEAR(s->indent); - Py_CLEAR(s->key_separator); - Py_CLEAR(s->item_separator); - Py_CLEAR(s->key_memo); - Py_CLEAR(s->skipkeys_bool); - Py_CLEAR(s->sort_keys); - Py_CLEAR(s->item_sort_kw); - Py_CLEAR(s->item_sort_key); - Py_CLEAR(s->max_long_size); - Py_CLEAR(s->min_long_size); - Py_CLEAR(s->Decimal); - return 0; -} - -PyDoc_STRVAR(encoder_doc, "_iterencode(obj, _current_indent_level) -> iterable"); - -static -PyTypeObject PyEncoderType = { - PyVarObject_HEAD_INIT(NULL, 0) - "simplejson._speedups.Encoder", /* tp_name */ - sizeof(PyEncoderObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - encoder_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - encoder_call, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - encoder_doc, /* tp_doc */ - encoder_traverse, /* tp_traverse */ - encoder_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - encoder_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - encoder_new, /* tp_new */ - 0, /* tp_free */ -}; - -static PyMethodDef speedups_methods[] = { - {"encode_basestring_ascii", - (PyCFunction)py_encode_basestring_ascii, - METH_O, - pydoc_encode_basestring_ascii}, - {"scanstring", - (PyCFunction)py_scanstring, - METH_VARARGS, - pydoc_scanstring}, - {NULL, NULL, 0, NULL} -}; - -PyDoc_STRVAR(module_doc, -"simplejson speedups\n"); - -#if PY_MAJOR_VERSION >= 3 -static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - "_speedups", /* m_name */ - module_doc, /* m_doc */ - -1, /* m_size */ - speedups_methods, /* m_methods */ - NULL, /* m_reload */ - NULL, /* m_traverse */ - NULL, /* m_clear*/ - NULL, /* m_free */ -}; -#endif - -PyObject * -import_dependency(char *module_name, char *attr_name) -{ - PyObject *rval; - PyObject *module = PyImport_ImportModule(module_name); - if (module == NULL) - return NULL; - rval = PyObject_GetAttrString(module, attr_name); - Py_DECREF(module); - return rval; -} - -static int -init_constants(void) -{ - JSON_NaN = JSON_InternFromString("NaN"); - if (JSON_NaN == NULL) - return 0; - JSON_Infinity = JSON_InternFromString("Infinity"); - if (JSON_Infinity == NULL) - return 0; - JSON_NegInfinity = JSON_InternFromString("-Infinity"); - if (JSON_NegInfinity == NULL) - return 0; -#if PY_MAJOR_VERSION >= 3 - JSON_EmptyUnicode = PyUnicode_New(0, 127); -#else /* PY_MAJOR_VERSION >= 3 */ - JSON_EmptyStr = PyString_FromString(""); - if (JSON_EmptyStr == NULL) - return 0; - JSON_EmptyUnicode = PyUnicode_FromUnicode(NULL, 0); -#endif /* PY_MAJOR_VERSION >= 3 */ - if (JSON_EmptyUnicode == NULL) - return 0; - - return 1; -} - -static PyObject * -moduleinit(void) -{ - PyObject *m; - if (PyType_Ready(&PyScannerType) < 0) - return NULL; - if (PyType_Ready(&PyEncoderType) < 0) - return NULL; - if (!init_constants()) - return NULL; - -#if PY_MAJOR_VERSION >= 3 - m = PyModule_Create(&moduledef); -#else - m = Py_InitModule3("_speedups", speedups_methods, module_doc); -#endif - Py_INCREF((PyObject*)&PyScannerType); - PyModule_AddObject(m, "make_scanner", (PyObject*)&PyScannerType); - Py_INCREF((PyObject*)&PyEncoderType); - PyModule_AddObject(m, "make_encoder", (PyObject*)&PyEncoderType); - RawJSONType = import_dependency("simplejson.raw_json", "RawJSON"); - if (RawJSONType == NULL) - return NULL; - JSONDecodeError = import_dependency("simplejson.errors", "JSONDecodeError"); - if (JSONDecodeError == NULL) - return NULL; - return m; -} - -#if PY_MAJOR_VERSION >= 3 -PyMODINIT_FUNC -PyInit__speedups(void) -{ - return moduleinit(); -} -#else -void -init_speedups(void) -{ - moduleinit(); -} -#endif diff --git a/contrib/python/simplejson/py3/simplejson/compat.py b/contrib/python/simplejson/py3/simplejson/compat.py deleted file mode 100644 index 5fc14128443..00000000000 --- a/contrib/python/simplejson/py3/simplejson/compat.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Python 3 compatibility shims -""" -import sys -if sys.version_info[0] < 3: - PY3 = False - def b(s): - return s - try: - from cStringIO import StringIO - except ImportError: - from StringIO import StringIO - BytesIO = StringIO - text_type = unicode - binary_type = str - string_types = (basestring,) - integer_types = (int, long) - unichr = unichr - reload_module = reload -else: - PY3 = True - if sys.version_info[:2] >= (3, 4): - from importlib import reload as reload_module - else: - from imp import reload as reload_module - def b(s): - return bytes(s, 'latin1') - from io import StringIO, BytesIO - text_type = str - binary_type = bytes - string_types = (str,) - integer_types = (int,) - unichr = chr - -long_type = integer_types[-1] diff --git a/contrib/python/simplejson/py3/simplejson/decoder.py b/contrib/python/simplejson/py3/simplejson/decoder.py deleted file mode 100644 index c99a976d84e..00000000000 --- a/contrib/python/simplejson/py3/simplejson/decoder.py +++ /dev/null @@ -1,416 +0,0 @@ -"""Implementation of JSONDecoder -""" -from __future__ import absolute_import -import re -import sys -import struct -from .compat import PY3, unichr -from .scanner import make_scanner, JSONDecodeError - -def _import_c_scanstring(): - try: - from ._speedups import scanstring - return scanstring - except ImportError: - return None -c_scanstring = _import_c_scanstring() - -# NOTE (3.1.0): JSONDecodeError may still be imported from this module for -# compatibility, but it was never in the __all__ -__all__ = ['JSONDecoder'] - -FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL - -def _floatconstants(): - if sys.version_info < (2, 6): - _BYTES = '7FF80000000000007FF0000000000000'.decode('hex') - nan, inf = struct.unpack('>dd', _BYTES) - else: - nan = float('nan') - inf = float('inf') - return nan, inf, -inf - -NaN, PosInf, NegInf = _floatconstants() - -_CONSTANTS = { - '-Infinity': NegInf, - 'Infinity': PosInf, - 'NaN': NaN, -} - -STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS) -BACKSLASH = { - '"': u'"', '\\': u'\\', '/': u'/', - 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t', -} - -DEFAULT_ENCODING = "utf-8" - -if hasattr(sys, 'get_int_max_str_digits'): - bounded_int = int -else: - def bounded_int(s, INT_MAX_STR_DIGITS=4300): - """Backport of the integer string length conversion limitation - - https://docs.python.org/3/library/stdtypes.html#int-max-str-digits - """ - if len(s) > INT_MAX_STR_DIGITS: - raise ValueError("Exceeds the limit (%s) for integer string conversion: value has %s digits" % (INT_MAX_STR_DIGITS, len(s))) - return int(s) - - -def scan_four_digit_hex(s, end, _m=re.compile(r'^[0-9a-fA-F]{4}$').match): - """Scan a four digit hex number from s[end:end + 4] - """ - msg = "Invalid \\uXXXX escape sequence" - esc = s[end:end + 4] - if not _m(esc): - raise JSONDecodeError(msg, s, end - 2) - try: - return int(esc, 16), end + 4 - except ValueError: - raise JSONDecodeError(msg, s, end - 2) - -def py_scanstring(s, end, encoding=None, strict=True, - _b=BACKSLASH, _m=STRINGCHUNK.match, _join=u''.join, - _PY3=PY3, _maxunicode=sys.maxunicode, - _scan_four_digit_hex=scan_four_digit_hex): - """Scan the string s for a JSON string. End is the index of the - character in s after the quote that started the JSON string. - Unescapes all valid JSON string escape sequences and raises ValueError - on attempt to decode an invalid string. If strict is False then literal - control characters are allowed in the string. - - Returns a tuple of the decoded string and the index of the character in s - after the end quote.""" - if encoding is None: - encoding = DEFAULT_ENCODING - chunks = [] - _append = chunks.append - begin = end - 1 - while 1: - chunk = _m(s, end) - if chunk is None: - raise JSONDecodeError( - "Unterminated string starting at", s, begin) - prev_end = end - end = chunk.end() - content, terminator = chunk.groups() - # Content is contains zero or more unescaped string characters - if content: - if not _PY3 and not isinstance(content, unicode): - content = unicode(content, encoding) - _append(content) - # Terminator is the end of string, a literal control character, - # or a backslash denoting that an escape sequence follows - if terminator == '"': - break - elif terminator != '\\': - if strict: - msg = "Invalid control character %r at" - raise JSONDecodeError(msg, s, prev_end) - else: - _append(terminator) - continue - try: - esc = s[end] - except IndexError: - raise JSONDecodeError( - "Unterminated string starting at", s, begin) - # If not a unicode escape sequence, must be in the lookup table - if esc != 'u': - try: - char = _b[esc] - except KeyError: - msg = "Invalid \\X escape sequence %r" - raise JSONDecodeError(msg, s, end) - end += 1 - else: - # Unicode escape sequence - uni, end = _scan_four_digit_hex(s, end + 1) - # Check for surrogate pair on UCS-4 systems - # Note that this will join high/low surrogate pairs - # but will also pass unpaired surrogates through - if (_maxunicode > 65535 and - uni & 0xfc00 == 0xd800 and - s[end:end + 2] == '\\u'): - uni2, end2 = _scan_four_digit_hex(s, end + 2) - if uni2 & 0xfc00 == 0xdc00: - uni = 0x10000 + (((uni - 0xd800) << 10) | - (uni2 - 0xdc00)) - end = end2 - char = unichr(uni) - # Append the unescaped character - _append(char) - return _join(chunks), end - - -# Use speedup if available -scanstring = c_scanstring or py_scanstring - -WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS) -WHITESPACE_STR = ' \t\n\r' - -def JSONObject(state, encoding, strict, scan_once, object_hook, - object_pairs_hook, memo=None, - _w=WHITESPACE.match, _ws=WHITESPACE_STR): - (s, end) = state - # Backwards compatibility - if memo is None: - memo = {} - memo_get = memo.setdefault - pairs = [] - # Use a slice to prevent IndexError from being raised, the following - # check will raise a more specific ValueError if the string is empty - nextchar = s[end:end + 1] - # Normally we expect nextchar == '"' - if nextchar != '"': - if nextchar in _ws: - end = _w(s, end).end() - nextchar = s[end:end + 1] - # Trivial empty object - if nextchar == '}': - if object_pairs_hook is not None: - result = object_pairs_hook(pairs) - return result, end + 1 - pairs = {} - if object_hook is not None: - pairs = object_hook(pairs) - return pairs, end + 1 - elif nextchar != '"': - raise JSONDecodeError( - "Expecting property name enclosed in double quotes or '}'", - s, end) - end += 1 - while True: - key, end = scanstring(s, end, encoding, strict) - key = memo_get(key, key) - - # To skip some function call overhead we optimize the fast paths where - # the JSON key separator is ": " or just ":". - if s[end:end + 1] != ':': - end = _w(s, end).end() - if s[end:end + 1] != ':': - raise JSONDecodeError("Expecting ':' delimiter", s, end) - - end += 1 - - try: - if s[end] in _ws: - end += 1 - if s[end] in _ws: - end = _w(s, end + 1).end() - except IndexError: - pass - - value, end = scan_once(s, end) - pairs.append((key, value)) - - try: - nextchar = s[end] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end] - except IndexError: - nextchar = '' - end += 1 - - if nextchar == '}': - break - elif nextchar != ',': - raise JSONDecodeError("Expecting ',' delimiter or '}'", s, end - 1) - - try: - nextchar = s[end] - if nextchar in _ws: - end += 1 - nextchar = s[end] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end] - except IndexError: - nextchar = '' - - end += 1 - if nextchar != '"': - raise JSONDecodeError( - "Expecting property name enclosed in double quotes", - s, end - 1) - - if object_pairs_hook is not None: - result = object_pairs_hook(pairs) - return result, end - pairs = dict(pairs) - if object_hook is not None: - pairs = object_hook(pairs) - return pairs, end - -def JSONArray(state, scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): - (s, end) = state - values = [] - nextchar = s[end:end + 1] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end:end + 1] - # Look-ahead for trivial empty array - if nextchar == ']': - return values, end + 1 - elif nextchar == '': - raise JSONDecodeError("Expecting value or ']'", s, end) - _append = values.append - while True: - value, end = scan_once(s, end) - _append(value) - nextchar = s[end:end + 1] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end:end + 1] - end += 1 - if nextchar == ']': - break - elif nextchar != ',': - raise JSONDecodeError("Expecting ',' delimiter or ']'", s, end - 1) - - try: - if s[end] in _ws: - end += 1 - if s[end] in _ws: - end = _w(s, end + 1).end() - except IndexError: - pass - - return values, end - -class JSONDecoder(object): - """Simple JSON <http://json.org> decoder - - Performs the following translations in decoding by default: - - +---------------+-------------------+ - | JSON | Python | - +===============+===================+ - | object | dict | - +---------------+-------------------+ - | array | list | - +---------------+-------------------+ - | string | str, unicode | - +---------------+-------------------+ - | number (int) | int, long | - +---------------+-------------------+ - | number (real) | float | - +---------------+-------------------+ - | true | True | - +---------------+-------------------+ - | false | False | - +---------------+-------------------+ - | null | None | - +---------------+-------------------+ - - When allow_nan=True, it also understands - ``NaN``, ``Infinity``, and ``-Infinity`` as - their corresponding ``float`` values, which is outside the JSON spec. - - """ - - def __init__(self, encoding=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, strict=True, - object_pairs_hook=None, allow_nan=False): - """ - *encoding* determines the encoding used to interpret any - :class:`str` objects decoded by this instance (``'utf-8'`` by - default). It has no effect when decoding :class:`unicode` objects. - - Note that currently only encodings that are a superset of ASCII work, - strings of other encodings should be passed in as :class:`unicode`. - - *object_hook*, if specified, will be called with the result of every - JSON object decoded and its return value will be used in place of the - given :class:`dict`. This can be used to provide custom - deserializations (e.g. to support JSON-RPC class hinting). - - *object_pairs_hook* is an optional function that will be called with - the result of any object literal decode with an ordered list of pairs. - The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders - that rely on the order that the key and value pairs are decoded (for - example, :func:`collections.OrderedDict` will remember the order of - insertion). If *object_hook* is also defined, the *object_pairs_hook* - takes priority. - - *parse_float*, if specified, will be called with the string of every - JSON float to be decoded. By default, this is equivalent to - ``float(num_str)``. This can be used to use another datatype or parser - for JSON floats (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every - JSON int to be decoded. By default, this is equivalent to - ``int(num_str)``. This can be used to use another datatype or parser - for JSON integers (e.g. :class:`float`). - - *allow_nan*, if True (default false), will allow the parser to - accept the non-standard floats ``NaN``, ``Infinity``, and ``-Infinity``. - - *parse_constant*, if specified, will be - called with one of the following strings: ``'-Infinity'``, - ``'Infinity'``, ``'NaN'``. It is not recommended to use this feature, - as it is rare to parse non-compliant JSON containing these values. - - *strict* controls the parser's behavior when it encounters an - invalid control character in a string. The default setting of - ``True`` means that unescaped control characters are parse errors, if - ``False`` then control characters will be allowed in strings. - - """ - if encoding is None: - encoding = DEFAULT_ENCODING - self.encoding = encoding - self.object_hook = object_hook - self.object_pairs_hook = object_pairs_hook - self.parse_float = parse_float or float - self.parse_int = parse_int or bounded_int - self.parse_constant = parse_constant or (allow_nan and _CONSTANTS.__getitem__ or None) - self.strict = strict - self.parse_object = JSONObject - self.parse_array = JSONArray - self.parse_string = scanstring - self.memo = {} - self.scan_once = make_scanner(self) - - def decode(self, s, _w=WHITESPACE.match, _PY3=PY3): - """Return the Python representation of ``s`` (a ``str`` or ``unicode`` - instance containing a JSON document) - - """ - if _PY3 and isinstance(s, bytes): - s = str(s, self.encoding) - obj, end = self.raw_decode(s) - end = _w(s, end).end() - if end != len(s): - raise JSONDecodeError("Extra data", s, end, len(s)) - return obj - - def raw_decode(self, s, idx=0, _w=WHITESPACE.match, _PY3=PY3): - """Decode a JSON document from ``s`` (a ``str`` or ``unicode`` - beginning with a JSON document) and return a 2-tuple of the Python - representation and the index in ``s`` where the document ended. - Optionally, ``idx`` can be used to specify an offset in ``s`` where - the JSON document begins. - - This can be used to decode a JSON document from a string that may - have extraneous data at the end. - - """ - if idx < 0: - # Ensure that raw_decode bails on negative indexes, the regex - # would otherwise mask this behavior. #98 - raise JSONDecodeError('Expecting value', s, idx) - if _PY3 and not isinstance(s, str): - raise TypeError("Input string must be text, not bytes") - # strip UTF-8 bom - if len(s) > idx: - ord0 = ord(s[idx]) - if ord0 == 0xfeff: - idx += 1 - elif ord0 == 0xef and s[idx:idx + 3] == '\xef\xbb\xbf': - idx += 3 - return self.scan_once(s, idx=_w(s, idx).end()) diff --git a/contrib/python/simplejson/py3/simplejson/encoder.py b/contrib/python/simplejson/py3/simplejson/encoder.py deleted file mode 100644 index 661ff361bf2..00000000000 --- a/contrib/python/simplejson/py3/simplejson/encoder.py +++ /dev/null @@ -1,740 +0,0 @@ -"""Implementation of JSONEncoder -""" -from __future__ import absolute_import -import re -from operator import itemgetter -# Do not import Decimal directly to avoid reload issues -import decimal -from .compat import binary_type, text_type, string_types, integer_types, PY3 -def _import_speedups(): - try: - from . import _speedups - return _speedups.encode_basestring_ascii, _speedups.make_encoder - except ImportError: - return None, None -c_encode_basestring_ascii, c_make_encoder = _import_speedups() - -from .decoder import PosInf -from .raw_json import RawJSON - -ESCAPE = re.compile(r'[\x00-\x1f\\"]') -ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') -HAS_UTF8 = re.compile(r'[\x80-\xff]') -ESCAPE_DCT = { - '\\': '\\\\', - '"': '\\"', - '\b': '\\b', - '\f': '\\f', - '\n': '\\n', - '\r': '\\r', - '\t': '\\t', -} -for i in range(0x20): - #ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i)) - ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,)) -del i - -FLOAT_REPR = repr - -def encode_basestring(s, _PY3=PY3, _q=u'"'): - """Return a JSON representation of a Python string - - """ - if _PY3: - if isinstance(s, bytes): - s = str(s, 'utf-8') - elif type(s) is not str: - # convert an str subclass instance to exact str - # raise a TypeError otherwise - s = str.__str__(s) - else: - if isinstance(s, str) and HAS_UTF8.search(s) is not None: - s = unicode(s, 'utf-8') - elif type(s) not in (str, unicode): - # convert an str subclass instance to exact str - # convert a unicode subclass instance to exact unicode - # raise a TypeError otherwise - if isinstance(s, str): - s = str.__str__(s) - else: - s = unicode.__getnewargs__(s)[0] - def replace(match): - return ESCAPE_DCT[match.group(0)] - return _q + ESCAPE.sub(replace, s) + _q - - -def py_encode_basestring_ascii(s, _PY3=PY3): - """Return an ASCII-only JSON representation of a Python string - - """ - if _PY3: - if isinstance(s, bytes): - s = str(s, 'utf-8') - elif type(s) is not str: - # convert an str subclass instance to exact str - # raise a TypeError otherwise - s = str.__str__(s) - else: - if isinstance(s, str) and HAS_UTF8.search(s) is not None: - s = unicode(s, 'utf-8') - elif type(s) not in (str, unicode): - # convert an str subclass instance to exact str - # convert a unicode subclass instance to exact unicode - # raise a TypeError otherwise - if isinstance(s, str): - s = str.__str__(s) - else: - s = unicode.__getnewargs__(s)[0] - def replace(match): - s = match.group(0) - try: - return ESCAPE_DCT[s] - except KeyError: - n = ord(s) - if n < 0x10000: - #return '\\u{0:04x}'.format(n) - return '\\u%04x' % (n,) - else: - # surrogate pair - n -= 0x10000 - s1 = 0xd800 | ((n >> 10) & 0x3ff) - s2 = 0xdc00 | (n & 0x3ff) - #return '\\u{0:04x}\\u{1:04x}'.format(s1, s2) - return '\\u%04x\\u%04x' % (s1, s2) - return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"' - - -encode_basestring_ascii = ( - c_encode_basestring_ascii or py_encode_basestring_ascii) - -class JSONEncoder(object): - """Extensible JSON <http://json.org> encoder for Python data structures. - - Supports the following objects and types by default: - - +-------------------+---------------+ - | Python | JSON | - +===================+===============+ - | dict, namedtuple | object | - +-------------------+---------------+ - | list, tuple | array | - +-------------------+---------------+ - | str, unicode | string | - +-------------------+---------------+ - | int, long, float | number | - +-------------------+---------------+ - | True | true | - +-------------------+---------------+ - | False | false | - +-------------------+---------------+ - | None | null | - +-------------------+---------------+ - - To extend this to recognize other objects, subclass and implement a - ``.default()`` method with another method that returns a serializable - object for ``o`` if possible, otherwise it should call the superclass - implementation (to raise ``TypeError``). - - """ - item_separator = ', ' - key_separator = ': ' - - def __init__(self, skipkeys=False, ensure_ascii=True, - check_circular=True, allow_nan=False, sort_keys=False, - indent=None, separators=None, encoding='utf-8', default=None, - use_decimal=True, namedtuple_as_object=True, - tuple_as_array=True, bigint_as_string=False, - item_sort_key=None, for_json=False, ignore_nan=False, - int_as_string_bitcount=None, iterable_as_array=False): - """Constructor for JSONEncoder, with sensible defaults. - - If skipkeys is false, then it is a TypeError to attempt - encoding of keys that are not str, int, long, float or None. If - skipkeys is True, such items are simply skipped. - - If ensure_ascii is true, the output is guaranteed to be str - objects with all incoming unicode characters escaped. If - ensure_ascii is false, the output will be unicode object. - - If check_circular is true, then lists, dicts, and custom encoded - objects will be checked for circular references during encoding to - prevent an infinite recursion (which would cause an OverflowError). - Otherwise, no such check takes place. - - If allow_nan is true (default: False), then out of range float - values (nan, inf, -inf) will be serialized to - their JavaScript equivalents (NaN, Infinity, -Infinity) - instead of raising a ValueError. See - ignore_nan for ECMA-262 compliant behavior. - - If sort_keys is true, then the output of dictionaries will be - sorted by key; this is useful for regression tests to ensure - that JSON serializations can be compared on a day-to-day basis. - - If indent is a string, then JSON array elements and object members - will be pretty-printed with a newline followed by that string repeated - for each level of nesting. ``None`` (the default) selects the most compact - representation without any newlines. For backwards compatibility with - versions of simplejson earlier than 2.1.0, an integer is also accepted - and is converted to a string with that many spaces. - - If specified, separators should be an (item_separator, key_separator) - tuple. The default is (', ', ': ') if *indent* is ``None`` and - (',', ': ') otherwise. To get the most compact JSON representation, - you should specify (',', ':') to eliminate whitespace. - - If specified, default is a function that gets called for objects - that can't otherwise be serialized. It should return a JSON encodable - version of the object or raise a ``TypeError``. - - If encoding is not None, then all input strings will be - transformed into unicode using that encoding prior to JSON-encoding. - The default is UTF-8. - - If use_decimal is true (default: ``True``), ``decimal.Decimal`` will - be supported directly by the encoder. For the inverse, decode JSON - with ``parse_float=decimal.Decimal``. - - If namedtuple_as_object is true (the default), objects with - ``_asdict()`` methods will be encoded as JSON objects. - - If tuple_as_array is true (the default), tuple (and subclasses) will - be encoded as JSON arrays. - - If *iterable_as_array* is true (default: ``False``), - any object not in the above table that implements ``__iter__()`` - will be encoded as a JSON array. - - If bigint_as_string is true (not the default), ints 2**53 and higher - or lower than -2**53 will be encoded as strings. This is to avoid the - rounding that happens in Javascript otherwise. - - If int_as_string_bitcount is a positive number (n), then int of size - greater than or equal to 2**n or lower than or equal to -2**n will be - encoded as strings. - - If specified, item_sort_key is a callable used to sort the items in - each dictionary. This is useful if you want to sort items other than - in alphabetical order by key. - - If for_json is true (not the default), objects with a ``for_json()`` - method will use the return value of that method for encoding as JSON - instead of the object. - - If *ignore_nan* is true (default: ``False``), then out of range - :class:`float` values (``nan``, ``inf``, ``-inf``) will be serialized - as ``null`` in compliance with the ECMA-262 specification. If true, - this will override *allow_nan*. - - """ - - self.skipkeys = skipkeys - self.ensure_ascii = ensure_ascii - self.check_circular = check_circular - self.allow_nan = allow_nan - self.sort_keys = sort_keys - self.use_decimal = use_decimal - self.namedtuple_as_object = namedtuple_as_object - self.tuple_as_array = tuple_as_array - self.iterable_as_array = iterable_as_array - self.bigint_as_string = bigint_as_string - self.item_sort_key = item_sort_key - self.for_json = for_json - self.ignore_nan = ignore_nan - self.int_as_string_bitcount = int_as_string_bitcount - if indent is not None and not isinstance(indent, string_types): - indent = indent * ' ' - self.indent = indent - if separators is not None: - self.item_separator, self.key_separator = separators - elif indent is not None: - self.item_separator = ',' - if default is not None: - self.default = default - self.encoding = encoding - - def default(self, o): - """Implement this method in a subclass such that it returns - a serializable object for ``o``, or calls the base implementation - (to raise a ``TypeError``). - - For example, to support arbitrary iterators, you could - implement default like this:: - - def default(self, o): - try: - iterable = iter(o) - except TypeError: - pass - else: - return list(iterable) - return JSONEncoder.default(self, o) - - """ - raise TypeError('Object of type %s is not JSON serializable' % - o.__class__.__name__) - - def encode(self, o): - """Return a JSON string representation of a Python data structure. - - >>> from simplejson import JSONEncoder - >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) - '{"foo": ["bar", "baz"]}' - - """ - # This is for extremely simple cases and benchmarks. - if isinstance(o, binary_type): - _encoding = self.encoding - if (_encoding is not None and not (_encoding == 'utf-8')): - o = text_type(o, _encoding) - if isinstance(o, string_types): - if self.ensure_ascii: - return encode_basestring_ascii(o) - else: - return encode_basestring(o) - # This doesn't pass the iterator directly to ''.join() because the - # exceptions aren't as detailed. The list call should be roughly - # equivalent to the PySequence_Fast that ''.join() would do. - chunks = self.iterencode(o) - if not isinstance(chunks, (list, tuple)): - chunks = list(chunks) - if self.ensure_ascii: - return ''.join(chunks) - else: - return u''.join(chunks) - - def iterencode(self, o): - """Encode the given object and yield each string - representation as available. - - For example:: - - for chunk in JSONEncoder().iterencode(bigobject): - mysocket.write(chunk) - - """ - if self.check_circular: - markers = {} - else: - markers = None - if self.ensure_ascii: - _encoder = encode_basestring_ascii - else: - _encoder = encode_basestring - if self.encoding != 'utf-8' and self.encoding is not None: - def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding): - if isinstance(o, binary_type): - o = text_type(o, _encoding) - return _orig_encoder(o) - - def floatstr(o, allow_nan=self.allow_nan, ignore_nan=self.ignore_nan, - _repr=FLOAT_REPR, _inf=PosInf, _neginf=-PosInf): - # Check for specials. Note that this type of test is processor - # and/or platform-specific, so do tests which don't depend on - # the internals. - - if o != o: - text = 'NaN' - elif o == _inf: - text = 'Infinity' - elif o == _neginf: - text = '-Infinity' - else: - if type(o) != float: - # See #118, do not trust custom str/repr - o = float(o) - return _repr(o) - - if ignore_nan: - text = 'null' - elif not allow_nan: - raise ValueError( - "Out of range float values are not JSON compliant: " + - repr(o)) - - return text - - key_memo = {} - int_as_string_bitcount = ( - 53 if self.bigint_as_string else self.int_as_string_bitcount) - if (c_make_encoder is not None and self.indent is None): - _iterencode = c_make_encoder( - markers, self.default, _encoder, self.indent, - self.key_separator, self.item_separator, self.sort_keys, - self.skipkeys, self.allow_nan, key_memo, self.use_decimal, - self.namedtuple_as_object, self.tuple_as_array, - int_as_string_bitcount, - self.item_sort_key, self.encoding, self.for_json, - self.ignore_nan, decimal.Decimal, self.iterable_as_array) - else: - _iterencode = _make_iterencode( - markers, self.default, _encoder, self.indent, floatstr, - self.key_separator, self.item_separator, self.sort_keys, - self.skipkeys, self.use_decimal, - self.namedtuple_as_object, self.tuple_as_array, - int_as_string_bitcount, - self.item_sort_key, self.encoding, self.for_json, - self.iterable_as_array, Decimal=decimal.Decimal) - try: - return _iterencode(o, 0) - finally: - key_memo.clear() - - -class JSONEncoderForHTML(JSONEncoder): - """An encoder that produces JSON safe to embed in HTML. - - To embed JSON content in, say, a script tag on a web page, the - characters &, < and > should be escaped. They cannot be escaped - with the usual entities (e.g. &) because they are not expanded - within <script> tags. - - This class also escapes the line separator and paragraph separator - characters U+2028 and U+2029, irrespective of the ensure_ascii setting, - as these characters are not valid in JavaScript strings (see - http://timelessrepo.com/json-isnt-a-javascript-subset). - """ - - def encode(self, o): - # Override JSONEncoder.encode because it has hacks for - # performance that make things more complicated. - chunks = self.iterencode(o) - if self.ensure_ascii: - return ''.join(chunks) - else: - return u''.join(chunks) - - def iterencode(self, o): - chunks = super(JSONEncoderForHTML, self).iterencode(o) - for chunk in chunks: - chunk = chunk.replace('&', '\\u0026') - chunk = chunk.replace('<', '\\u003c') - chunk = chunk.replace('>', '\\u003e') - - if not self.ensure_ascii: - chunk = chunk.replace(u'\u2028', '\\u2028') - chunk = chunk.replace(u'\u2029', '\\u2029') - - yield chunk - - -def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, - _key_separator, _item_separator, _sort_keys, _skipkeys, - _use_decimal, _namedtuple_as_object, _tuple_as_array, - _int_as_string_bitcount, _item_sort_key, - _encoding,_for_json, - _iterable_as_array, - ## HACK: hand-optimized bytecode; turn globals into locals - _PY3=PY3, - ValueError=ValueError, - string_types=string_types, - Decimal=None, - dict=dict, - float=float, - id=id, - integer_types=integer_types, - isinstance=isinstance, - list=list, - str=str, - tuple=tuple, - iter=iter, - ): - if _use_decimal and Decimal is None: - Decimal = decimal.Decimal - if _item_sort_key and not callable(_item_sort_key): - raise TypeError("item_sort_key must be None or callable") - elif _sort_keys and not _item_sort_key: - _item_sort_key = itemgetter(0) - - if (_int_as_string_bitcount is not None and - (_int_as_string_bitcount <= 0 or - not isinstance(_int_as_string_bitcount, integer_types))): - raise TypeError("int_as_string_bitcount must be a positive integer") - - def call_method(obj, method_name): - method = getattr(obj, method_name, None) - if callable(method): - try: - return (method(),) - except TypeError: - pass - return None - - def _encode_int(value): - skip_quoting = ( - _int_as_string_bitcount is None - or - _int_as_string_bitcount < 1 - ) - if type(value) not in integer_types: - # See #118, do not trust custom str/repr - value = int(value) - if ( - skip_quoting or - (-1 << _int_as_string_bitcount) - < value < - (1 << _int_as_string_bitcount) - ): - return str(value) - return '"' + str(value) + '"' - - def _iterencode_list(lst, _current_indent_level): - if not lst: - yield '[]' - return - if markers is not None: - markerid = id(lst) - if markerid in markers: - raise ValueError("Circular reference detected") - markers[markerid] = lst - buf = '[' - if _indent is not None: - _current_indent_level += 1 - newline_indent = '\n' + (_indent * _current_indent_level) - separator = _item_separator + newline_indent - buf += newline_indent - else: - newline_indent = None - separator = _item_separator - first = True - for value in lst: - if first: - first = False - else: - buf = separator - if isinstance(value, string_types): - yield buf + _encoder(value) - elif _PY3 and isinstance(value, bytes) and _encoding is not None: - yield buf + _encoder(value) - elif isinstance(value, RawJSON): - yield buf + value.encoded_json - elif value is None: - yield buf + 'null' - elif value is True: - yield buf + 'true' - elif value is False: - yield buf + 'false' - elif isinstance(value, integer_types): - yield buf + _encode_int(value) - elif isinstance(value, float): - yield buf + _floatstr(value) - elif _use_decimal and isinstance(value, Decimal): - yield buf + str(value) - else: - yield buf - for_json = _for_json and call_method(value, 'for_json') - if for_json: - chunks = _iterencode(for_json[0], _current_indent_level) - elif isinstance(value, list): - chunks = _iterencode_list(value, _current_indent_level) - else: - _asdict = _namedtuple_as_object and call_method(value, '_asdict') - if _asdict: - dct = _asdict[0] - if not isinstance(dct, dict): - raise TypeError("_asdict() must return a dict, not %s" % (type(dct).__name__,)) - chunks = _iterencode_dict(dct, - _current_indent_level) - elif _tuple_as_array and isinstance(value, tuple): - chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): - chunks = _iterencode_dict(value, _current_indent_level) - else: - chunks = _iterencode(value, _current_indent_level) - for chunk in chunks: - yield chunk - if first: - # iterable_as_array misses the fast path at the top - yield '[]' - else: - if newline_indent is not None: - _current_indent_level -= 1 - yield '\n' + (_indent * _current_indent_level) - yield ']' - if markers is not None: - del markers[markerid] - - def _stringify_key(key): - if isinstance(key, string_types): # pragma: no cover - pass - elif _PY3 and isinstance(key, bytes) and _encoding is not None: - key = str(key, _encoding) - elif isinstance(key, float): - key = _floatstr(key) - elif key is True: - key = 'true' - elif key is False: - key = 'false' - elif key is None: - key = 'null' - elif isinstance(key, integer_types): - if type(key) not in integer_types: - # See #118, do not trust custom str/repr - key = int(key) - key = str(key) - elif _use_decimal and isinstance(key, Decimal): - key = str(key) - elif _skipkeys: - key = None - else: - raise TypeError('keys must be str, int, float, bool or None, ' - 'not %s' % key.__class__.__name__) - return key - - def _iterencode_dict(dct, _current_indent_level): - if not dct: - yield '{}' - return - if markers is not None: - markerid = id(dct) - if markerid in markers: - raise ValueError("Circular reference detected") - markers[markerid] = dct - yield '{' - if _indent is not None: - _current_indent_level += 1 - newline_indent = '\n' + (_indent * _current_indent_level) - item_separator = _item_separator + newline_indent - yield newline_indent - else: - newline_indent = None - item_separator = _item_separator - first = True - if _PY3: - iteritems = dct.items() - else: - iteritems = dct.iteritems() - if _item_sort_key: - items = [] - for k, v in dct.items(): - if not isinstance(k, string_types): - k = _stringify_key(k) - if k is None: - continue - items.append((k, v)) - items.sort(key=_item_sort_key) - else: - items = iteritems - for key, value in items: - if not (_item_sort_key or isinstance(key, string_types)): - key = _stringify_key(key) - if key is None: - # _skipkeys must be True - continue - if first: - first = False - else: - yield item_separator - yield _encoder(key) - yield _key_separator - if isinstance(value, string_types): - yield _encoder(value) - elif _PY3 and isinstance(value, bytes) and _encoding is not None: - yield _encoder(value) - elif isinstance(value, RawJSON): - yield value.encoded_json - elif value is None: - yield 'null' - elif value is True: - yield 'true' - elif value is False: - yield 'false' - elif isinstance(value, integer_types): - yield _encode_int(value) - elif isinstance(value, float): - yield _floatstr(value) - elif _use_decimal and isinstance(value, Decimal): - yield str(value) - else: - for_json = _for_json and call_method(value, 'for_json') - if for_json: - chunks = _iterencode(for_json[0], _current_indent_level) - elif isinstance(value, list): - chunks = _iterencode_list(value, _current_indent_level) - else: - _asdict = _namedtuple_as_object and call_method(value, '_asdict') - if _asdict: - dct = _asdict[0] - if not isinstance(dct, dict): - raise TypeError("_asdict() must return a dict, not %s" % (type(dct).__name__,)) - chunks = _iterencode_dict(dct, - _current_indent_level) - elif _tuple_as_array and isinstance(value, tuple): - chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): - chunks = _iterencode_dict(value, _current_indent_level) - else: - chunks = _iterencode(value, _current_indent_level) - for chunk in chunks: - yield chunk - if newline_indent is not None: - _current_indent_level -= 1 - yield '\n' + (_indent * _current_indent_level) - yield '}' - if markers is not None: - del markers[markerid] - - def _iterencode(o, _current_indent_level): - if isinstance(o, string_types): - yield _encoder(o) - elif _PY3 and isinstance(o, bytes) and _encoding is not None: - yield _encoder(o) - elif isinstance(o, RawJSON): - yield o.encoded_json - elif o is None: - yield 'null' - elif o is True: - yield 'true' - elif o is False: - yield 'false' - elif isinstance(o, integer_types): - yield _encode_int(o) - elif isinstance(o, float): - yield _floatstr(o) - else: - for_json = _for_json and call_method(o, 'for_json') - if for_json: - for chunk in _iterencode(for_json[0], _current_indent_level): - yield chunk - elif isinstance(o, list): - for chunk in _iterencode_list(o, _current_indent_level): - yield chunk - else: - _asdict = _namedtuple_as_object and call_method(o, '_asdict') - if _asdict: - dct = _asdict[0] - if not isinstance(dct, dict): - raise TypeError("_asdict() must return a dict, not %s" % (type(dct).__name__,)) - for chunk in _iterencode_dict(dct, _current_indent_level): - yield chunk - elif (_tuple_as_array and isinstance(o, tuple)): - for chunk in _iterencode_list(o, _current_indent_level): - yield chunk - elif isinstance(o, dict): - for chunk in _iterencode_dict(o, _current_indent_level): - yield chunk - elif _use_decimal and isinstance(o, Decimal): - yield str(o) - else: - while _iterable_as_array: - # Markers are not checked here because it is valid for - # an iterable to return self. - try: - o = iter(o) - except TypeError: - break - for chunk in _iterencode_list(o, _current_indent_level): - yield chunk - return - if markers is not None: - markerid = id(o) - if markerid in markers: - raise ValueError("Circular reference detected") - markers[markerid] = o - o = _default(o) - for chunk in _iterencode(o, _current_indent_level): - yield chunk - if markers is not None: - del markers[markerid] - - return _iterencode diff --git a/contrib/python/simplejson/py3/simplejson/errors.py b/contrib/python/simplejson/py3/simplejson/errors.py deleted file mode 100644 index b97ab1e913f..00000000000 --- a/contrib/python/simplejson/py3/simplejson/errors.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Error classes used by simplejson -""" -__all__ = ['JSONDecodeError'] - - -def linecol(doc, pos): - lineno = doc.count('\n', 0, pos) + 1 - if lineno == 1: - colno = pos + 1 - else: - colno = pos - doc.rindex('\n', 0, pos) - return lineno, colno - - -def errmsg(msg, doc, pos, end=None): - lineno, colno = linecol(doc, pos) - msg = msg.replace('%r', repr(doc[pos:pos + 1])) - if end is None: - fmt = '%s: line %d column %d (char %d)' - return fmt % (msg, lineno, colno, pos) - endlineno, endcolno = linecol(doc, end) - fmt = '%s: line %d column %d - line %d column %d (char %d - %d)' - return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end) - - -class JSONDecodeError(ValueError): - """Subclass of ValueError with the following additional properties: - - msg: The unformatted error message - doc: The JSON document being parsed - pos: The start index of doc where parsing failed - end: The end index of doc where parsing failed (may be None) - lineno: The line corresponding to pos - colno: The column corresponding to pos - endlineno: The line corresponding to end (may be None) - endcolno: The column corresponding to end (may be None) - - """ - # Note that this exception is used from _speedups - def __init__(self, msg, doc, pos, end=None): - ValueError.__init__(self, errmsg(msg, doc, pos, end=end)) - self.msg = msg - self.doc = doc - self.pos = pos - self.end = end - self.lineno, self.colno = linecol(doc, pos) - if end is not None: - self.endlineno, self.endcolno = linecol(doc, end) - else: - self.endlineno, self.endcolno = None, None - - def __reduce__(self): - return self.__class__, (self.msg, self.doc, self.pos, self.end) diff --git a/contrib/python/simplejson/py3/simplejson/ordered_dict.py b/contrib/python/simplejson/py3/simplejson/ordered_dict.py deleted file mode 100644 index d5a55ebd0f7..00000000000 --- a/contrib/python/simplejson/py3/simplejson/ordered_dict.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Drop-in replacement for collections.OrderedDict by Raymond Hettinger - -http://code.activestate.com/recipes/576693/ - -""" -from UserDict import DictMixin - -class OrderedDict(dict, DictMixin): - - def __init__(self, *args, **kwds): - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__end - except AttributeError: - self.clear() - self.update(*args, **kwds) - - def clear(self): - self.__end = end = [] - end += [None, end, end] # sentinel node for doubly linked list - self.__map = {} # key --> [key, prev, next] - dict.clear(self) - - def __setitem__(self, key, value): - if key not in self: - end = self.__end - curr = end[1] - curr[2] = end[1] = self.__map[key] = [key, curr, end] - dict.__setitem__(self, key, value) - - def __delitem__(self, key): - dict.__delitem__(self, key) - key, prev, next = self.__map.pop(key) - prev[2] = next - next[1] = prev - - def __iter__(self): - end = self.__end - curr = end[2] - while curr is not end: - yield curr[0] - curr = curr[2] - - def __reversed__(self): - end = self.__end - curr = end[1] - while curr is not end: - yield curr[0] - curr = curr[1] - - def popitem(self, last=True): - if not self: - raise KeyError('dictionary is empty') - key = reversed(self).next() if last else iter(self).next() - value = self.pop(key) - return key, value - - def __reduce__(self): - items = [[k, self[k]] for k in self] - tmp = self.__map, self.__end - del self.__map, self.__end - inst_dict = vars(self).copy() - self.__map, self.__end = tmp - if inst_dict: - return (self.__class__, (items,), inst_dict) - return self.__class__, (items,) - - def keys(self): - return list(self) - - setdefault = DictMixin.setdefault - update = DictMixin.update - pop = DictMixin.pop - values = DictMixin.values - items = DictMixin.items - iterkeys = DictMixin.iterkeys - itervalues = DictMixin.itervalues - iteritems = DictMixin.iteritems - - def __repr__(self): - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, self.items()) - - def copy(self): - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - d = cls() - for key in iterable: - d[key] = value - return d - - def __eq__(self, other): - if isinstance(other, OrderedDict): - return len(self)==len(other) and \ - all(p==q for p, q in zip(self.items(), other.items())) - return dict.__eq__(self, other) - - def __ne__(self, other): - return not self == other diff --git a/contrib/python/simplejson/py3/simplejson/raw_json.py b/contrib/python/simplejson/py3/simplejson/raw_json.py deleted file mode 100644 index 2071a70206b..00000000000 --- a/contrib/python/simplejson/py3/simplejson/raw_json.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Implementation of RawJSON -""" - -class RawJSON(object): - """Wrap an encoded JSON document for direct embedding in the output - - """ - def __init__(self, encoded_json): - self.encoded_json = encoded_json diff --git a/contrib/python/simplejson/py3/simplejson/scanner.py b/contrib/python/simplejson/py3/simplejson/scanner.py deleted file mode 100644 index 34710d68c6d..00000000000 --- a/contrib/python/simplejson/py3/simplejson/scanner.py +++ /dev/null @@ -1,85 +0,0 @@ -"""JSON token scanner -""" -import re -from .errors import JSONDecodeError -def _import_c_make_scanner(): - try: - from ._speedups import make_scanner - return make_scanner - except ImportError: - return None -c_make_scanner = _import_c_make_scanner() - -__all__ = ['make_scanner', 'JSONDecodeError'] - -NUMBER_RE = re.compile( - r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?', - (re.VERBOSE | re.MULTILINE | re.DOTALL)) - - -def py_make_scanner(context): - parse_object = context.parse_object - parse_array = context.parse_array - parse_string = context.parse_string - match_number = NUMBER_RE.match - encoding = context.encoding - strict = context.strict - parse_float = context.parse_float - parse_int = context.parse_int - parse_constant = context.parse_constant - object_hook = context.object_hook - object_pairs_hook = context.object_pairs_hook - memo = context.memo - - def _scan_once(string, idx): - errmsg = 'Expecting value' - try: - nextchar = string[idx] - except IndexError: - raise JSONDecodeError(errmsg, string, idx) - - if nextchar == '"': - return parse_string(string, idx + 1, encoding, strict) - elif nextchar == '{': - return parse_object((string, idx + 1), encoding, strict, - _scan_once, object_hook, object_pairs_hook, memo) - elif nextchar == '[': - return parse_array((string, idx + 1), _scan_once) - elif nextchar == 'n' and string[idx:idx + 4] == 'null': - return None, idx + 4 - elif nextchar == 't' and string[idx:idx + 4] == 'true': - return True, idx + 4 - elif nextchar == 'f' and string[idx:idx + 5] == 'false': - return False, idx + 5 - - m = match_number(string, idx) - if m is not None: - integer, frac, exp = m.groups() - if frac or exp: - res = parse_float(integer + (frac or '') + (exp or '')) - else: - res = parse_int(integer) - return res, m.end() - elif parse_constant and nextchar == 'N' and string[idx:idx + 3] == 'NaN': - return parse_constant('NaN'), idx + 3 - elif parse_constant and nextchar == 'I' and string[idx:idx + 8] == 'Infinity': - return parse_constant('Infinity'), idx + 8 - elif parse_constant and nextchar == '-' and string[idx:idx + 9] == '-Infinity': - return parse_constant('-Infinity'), idx + 9 - else: - raise JSONDecodeError(errmsg, string, idx) - - def scan_once(string, idx): - if idx < 0: - # Ensure the same behavior as the C speedup, otherwise - # this would work for *some* negative string indices due - # to the behavior of __getitem__ for strings. #98 - raise JSONDecodeError('Expecting value', string, idx) - try: - return _scan_once(string, idx) - finally: - memo.clear() - - return scan_once - -make_scanner = c_make_scanner or py_make_scanner diff --git a/contrib/python/simplejson/py3/simplejson/tests/__init__.py b/contrib/python/simplejson/py3/simplejson/tests/__init__.py deleted file mode 100644 index d9986d033a0..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/__init__.py +++ /dev/null @@ -1,93 +0,0 @@ -from __future__ import absolute_import -import unittest -import sys -import os - - -class NoExtensionTestSuite(unittest.TestSuite): - def run(self, result): - import simplejson - - simplejson._toggle_speedups(False) - result = unittest.TestSuite.run(self, result) - simplejson._toggle_speedups(True) - return result - - -class TestMissingSpeedups(unittest.TestCase): - def runTest(self): - if hasattr(sys, "pypy_translation_info"): - "PyPy doesn't need speedups! :)" - elif getattr(getattr(sys, "implementation", None), "name", None) == "graalpy": - "GraalPy doesn't need speedups! :)" - elif hasattr(self, "skipTest"): - self.skipTest("_speedups.so is missing!") - - -def additional_tests(suite=None, project_dir=None): - import simplejson - import simplejson.encoder - import simplejson.decoder - - if suite is None: - suite = unittest.TestSuite() - try: - import doctest - except ImportError: - if sys.version_info < (2, 7): - # doctests in 2.6 depends on cStringIO - return suite - raise - for mod in (simplejson, simplejson.encoder, simplejson.decoder): - suite.addTest(doctest.DocTestSuite(mod)) - if project_dir is not None: - suite.addTest( - doctest.DocFileSuite( - os.path.join(project_dir, "index.rst"), module_relative=False - ) - ) - return suite - - -def all_tests_suite(project_dir=None): - def get_suite(): - suite_names = [ - "simplejson.tests.%s" % (os.path.splitext(f)[0],) - for f in os.listdir(os.path.dirname(__file__)) - if f.startswith("test_") and f.endswith(".py") - ] - return additional_tests( - suite=unittest.TestLoader().loadTestsFromNames(suite_names), - project_dir=project_dir, - ) - - suite = get_suite() - import simplejson - - if simplejson._import_c_make_encoder() is None: - suite.addTest(TestMissingSpeedups()) - else: - suite = unittest.TestSuite( - [ - suite, - NoExtensionTestSuite([get_suite()]), - ] - ) - return suite - - -def main(project_dir=None): - runner = unittest.TextTestRunner(verbosity=1 + sys.argv.count("-v")) - suite = all_tests_suite(project_dir=project_dir) - raise SystemExit(not runner.run(suite).wasSuccessful()) - - -if __name__ == "__main__": - import os - import sys - - project_dir = os.path.dirname( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - ) - sys.path.insert(0, project_dir) - main(project_dir=project_dir) diff --git a/contrib/python/simplejson/py3/simplejson/tests/_cibw_runner.py b/contrib/python/simplejson/py3/simplejson/tests/_cibw_runner.py deleted file mode 100644 index bffb70b8add..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/_cibw_runner.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Internal module for running tests from cibuildwheel""" - -import sys -import simplejson.tests - -if __name__ == '__main__': - simplejson.tests.main(project_dir=sys.argv[1]) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_bigint_as_string.py b/contrib/python/simplejson/py3/simplejson/tests/test_bigint_as_string.py deleted file mode 100644 index 2cf2cc2966e..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_bigint_as_string.py +++ /dev/null @@ -1,67 +0,0 @@ -from unittest import TestCase - -import simplejson as json - - -class TestBigintAsString(TestCase): - # Python 2.5, at least the one that ships on Mac OS X, calculates - # 2 ** 53 as 0! It manages to calculate 1 << 53 correctly. - values = [(200, 200), - ((1 << 53) - 1, 9007199254740991), - ((1 << 53), '9007199254740992'), - ((1 << 53) + 1, '9007199254740993'), - (-100, -100), - ((-1 << 53), '-9007199254740992'), - ((-1 << 53) - 1, '-9007199254740993'), - ((-1 << 53) + 1, -9007199254740991)] - - options = ( - {"bigint_as_string": True}, - {"int_as_string_bitcount": 53} - ) - - def test_ints(self): - for opts in self.options: - for val, expect in self.values: - self.assertEqual( - val, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, **opts))) - - def test_lists(self): - for opts in self.options: - for val, expect in self.values: - val = [val, val] - expect = [expect, expect] - self.assertEqual( - val, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, **opts))) - - def test_dicts(self): - for opts in self.options: - for val, expect in self.values: - val = {'k': val} - expect = {'k': expect} - self.assertEqual( - val, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, **opts))) - - def test_dict_keys(self): - for opts in self.options: - for val, _ in self.values: - expect = {str(val): 'value'} - val = {val: 'value'} - self.assertEqual( - expect, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, **opts))) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_bitsize_int_as_string.py b/contrib/python/simplejson/py3/simplejson/tests/test_bitsize_int_as_string.py deleted file mode 100644 index fd7d103a6d8..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_bitsize_int_as_string.py +++ /dev/null @@ -1,73 +0,0 @@ -from unittest import TestCase - -import simplejson as json - - -class TestBitSizeIntAsString(TestCase): - # Python 2.5, at least the one that ships on Mac OS X, calculates - # 2 ** 31 as 0! It manages to calculate 1 << 31 correctly. - values = [ - (200, 200), - ((1 << 31) - 1, (1 << 31) - 1), - ((1 << 31), str(1 << 31)), - ((1 << 31) + 1, str((1 << 31) + 1)), - (-100, -100), - ((-1 << 31), str(-1 << 31)), - ((-1 << 31) - 1, str((-1 << 31) - 1)), - ((-1 << 31) + 1, (-1 << 31) + 1), - ] - - def test_invalid_counts(self): - for n in ['foo', -1, 0, 1.0]: - self.assertRaises( - TypeError, - json.dumps, 0, int_as_string_bitcount=n) - - def test_ints_outside_range_fails(self): - self.assertNotEqual( - str(1 << 15), - json.loads(json.dumps(1 << 15, int_as_string_bitcount=16)), - ) - - def test_ints(self): - for val, expect in self.values: - self.assertEqual( - val, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, int_as_string_bitcount=31)), - ) - - def test_lists(self): - for val, expect in self.values: - val = [val, val] - expect = [expect, expect] - self.assertEqual( - val, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, int_as_string_bitcount=31))) - - def test_dicts(self): - for val, expect in self.values: - val = {'k': val} - expect = {'k': expect} - self.assertEqual( - val, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, int_as_string_bitcount=31))) - - def test_dict_keys(self): - for val, _ in self.values: - expect = {str(val): 'value'} - val = {val: 'value'} - self.assertEqual( - expect, - json.loads(json.dumps(val))) - self.assertEqual( - expect, - json.loads(json.dumps(val, int_as_string_bitcount=31))) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_check_circular.py b/contrib/python/simplejson/py3/simplejson/tests/test_check_circular.py deleted file mode 100644 index af6463d6d88..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_check_circular.py +++ /dev/null @@ -1,30 +0,0 @@ -from unittest import TestCase -import simplejson as json - -def default_iterable(obj): - return list(obj) - -class TestCheckCircular(TestCase): - def test_circular_dict(self): - dct = {} - dct['a'] = dct - self.assertRaises(ValueError, json.dumps, dct) - - def test_circular_list(self): - lst = [] - lst.append(lst) - self.assertRaises(ValueError, json.dumps, lst) - - def test_circular_composite(self): - dct2 = {} - dct2['a'] = [] - dct2['a'].append(dct2) - self.assertRaises(ValueError, json.dumps, dct2) - - def test_circular_default(self): - json.dumps([set()], default=default_iterable) - self.assertRaises(TypeError, json.dumps, [set()]) - - def test_circular_off_default(self): - json.dumps([set()], default=default_iterable, check_circular=False) - self.assertRaises(TypeError, json.dumps, [set()], check_circular=False) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_decimal.py b/contrib/python/simplejson/py3/simplejson/tests/test_decimal.py deleted file mode 100644 index 2b0940b1527..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_decimal.py +++ /dev/null @@ -1,71 +0,0 @@ -import decimal -from decimal import Decimal -from unittest import TestCase -from simplejson.compat import StringIO, reload_module - -import simplejson as json - -class TestDecimal(TestCase): - NUMS = "1.0", "10.00", "1.1", "1234567890.1234567890", "500" - def dumps(self, obj, **kw): - sio = StringIO() - json.dump(obj, sio, **kw) - res = json.dumps(obj, **kw) - self.assertEqual(res, sio.getvalue()) - return res - - def loads(self, s, **kw): - sio = StringIO(s) - res = json.loads(s, **kw) - self.assertEqual(res, json.load(sio, **kw)) - return res - - def test_decimal_encode(self): - for d in map(Decimal, self.NUMS): - self.assertEqual(self.dumps(d, use_decimal=True), str(d)) - - def test_decimal_decode(self): - for s in self.NUMS: - self.assertEqual(self.loads(s, parse_float=Decimal), Decimal(s)) - - def test_stringify_key(self): - for d in map(Decimal, self.NUMS): - v = {d: d} - self.assertEqual( - self.loads( - self.dumps(v, use_decimal=True), parse_float=Decimal), - {str(d): d}) - - def test_decimal_roundtrip(self): - for d in map(Decimal, self.NUMS): - # The type might not be the same (int and Decimal) but they - # should still compare equal. - for v in [d, [d], {'': d}]: - self.assertEqual( - self.loads( - self.dumps(v, use_decimal=True), parse_float=Decimal), - v) - - def test_decimal_defaults(self): - d = Decimal('1.1') - # use_decimal=True is the default - self.assertRaises(TypeError, json.dumps, d, use_decimal=False) - self.assertEqual('1.1', json.dumps(d)) - self.assertEqual('1.1', json.dumps(d, use_decimal=True)) - self.assertRaises(TypeError, json.dump, d, StringIO(), - use_decimal=False) - sio = StringIO() - json.dump(d, sio) - self.assertEqual('1.1', sio.getvalue()) - sio = StringIO() - json.dump(d, sio, use_decimal=True) - self.assertEqual('1.1', sio.getvalue()) - - def test_decimal_reload(self): - # Simulate a subinterpreter that reloads the Python modules but not - # the C code https://github.com/simplejson/simplejson/issues/34 - global Decimal - Decimal = reload_module(decimal).Decimal - import simplejson.encoder - simplejson.encoder.Decimal = Decimal - self.test_decimal_roundtrip() diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_decode.py b/contrib/python/simplejson/py3/simplejson/tests/test_decode.py deleted file mode 100644 index 317b4f98d55..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_decode.py +++ /dev/null @@ -1,127 +0,0 @@ -from __future__ import absolute_import -import decimal -from unittest import TestCase - -import sys -import simplejson as json -from simplejson.compat import StringIO, b, binary_type -from simplejson import OrderedDict - -class MisbehavingBytesSubtype(binary_type): - def decode(self, encoding=None): - return "bad decode" - def __str__(self): - return "bad __str__" - def __bytes__(self): - return b("bad __bytes__") - -class TestDecode(TestCase): - if not hasattr(TestCase, 'assertIs'): - def assertIs(self, a, b): - self.assertTrue(a is b, '%r is %r' % (a, b)) - - def test_decimal(self): - rval = json.loads('1.1', parse_float=decimal.Decimal) - self.assertTrue(isinstance(rval, decimal.Decimal)) - self.assertEqual(rval, decimal.Decimal('1.1')) - - def test_float(self): - rval = json.loads('1', parse_int=float) - self.assertTrue(isinstance(rval, float)) - self.assertEqual(rval, 1.0) - - def test_decoder_optimizations(self): - # Several optimizations were made that skip over calls to - # the whitespace regex, so this test is designed to try and - # exercise the uncommon cases. The array cases are already covered. - rval = json.loads('{ "key" : "value" , "k":"v" }') - self.assertEqual(rval, {"key":"value", "k":"v"}) - - def test_empty_objects(self): - s = '{}' - self.assertEqual(json.loads(s), eval(s)) - s = '[]' - self.assertEqual(json.loads(s), eval(s)) - s = '""' - self.assertEqual(json.loads(s), eval(s)) - - def test_object_pairs_hook(self): - s = '{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}' - p = [("xkd", 1), ("kcw", 2), ("art", 3), ("hxm", 4), - ("qrt", 5), ("pad", 6), ("hoy", 7)] - self.assertEqual(json.loads(s), eval(s)) - self.assertEqual(json.loads(s, object_pairs_hook=lambda x: x), p) - self.assertEqual(json.load(StringIO(s), - object_pairs_hook=lambda x: x), p) - od = json.loads(s, object_pairs_hook=OrderedDict) - self.assertEqual(od, OrderedDict(p)) - self.assertEqual(type(od), OrderedDict) - # the object_pairs_hook takes priority over the object_hook - self.assertEqual(json.loads(s, - object_pairs_hook=OrderedDict, - object_hook=lambda x: None), - OrderedDict(p)) - - def check_keys_reuse(self, source, loads): - rval = loads(source) - (a, b), (c, d) = sorted(rval[0]), sorted(rval[1]) - self.assertIs(a, c) - self.assertIs(b, d) - - def test_keys_reuse_str(self): - s = u'[{"a_key": 1, "b_\xe9": 2}, {"a_key": 3, "b_\xe9": 4}]'.encode('utf8') - self.check_keys_reuse(s, json.loads) - - def test_keys_reuse_unicode(self): - s = u'[{"a_key": 1, "b_\xe9": 2}, {"a_key": 3, "b_\xe9": 4}]' - self.check_keys_reuse(s, json.loads) - - def test_empty_strings(self): - self.assertEqual(json.loads('""'), "") - self.assertEqual(json.loads(u'""'), u"") - self.assertEqual(json.loads('[""]'), [""]) - self.assertEqual(json.loads(u'[""]'), [u""]) - - def test_raw_decode(self): - cls = json.decoder.JSONDecoder - self.assertEqual( - ({'a': {}}, 9), - cls().raw_decode("{\"a\": {}}")) - # http://code.google.com/p/simplejson/issues/detail?id=85 - self.assertEqual( - ({'a': {}}, 9), - cls(object_pairs_hook=dict).raw_decode("{\"a\": {}}")) - # https://github.com/simplejson/simplejson/pull/38 - self.assertEqual( - ({'a': {}}, 11), - cls().raw_decode(" \n{\"a\": {}}")) - - def test_bytes_decode(self): - cls = json.decoder.JSONDecoder - data = b('"\xe2\x82\xac"') - self.assertEqual(cls().decode(data), u'\u20ac') - self.assertEqual(cls(encoding='latin1').decode(data), u'\xe2\x82\xac') - self.assertEqual(cls(encoding=None).decode(data), u'\u20ac') - - data = MisbehavingBytesSubtype(b('"\xe2\x82\xac"')) - self.assertEqual(cls().decode(data), u'\u20ac') - self.assertEqual(cls(encoding='latin1').decode(data), u'\xe2\x82\xac') - self.assertEqual(cls(encoding=None).decode(data), u'\u20ac') - - def test_bounds_checking(self): - # https://github.com/simplejson/simplejson/issues/98 - j = json.decoder.JSONDecoder() - for i in [4, 5, 6, -1, -2, -3, -4, -5, -6]: - self.assertRaises(ValueError, j.scan_once, '1234', i) - self.assertRaises(ValueError, j.raw_decode, '1234', i) - x, y = sorted(['128931233', '472389423'], key=id) - diff = id(x) - id(y) - self.assertRaises(ValueError, j.scan_once, y, diff) - self.assertRaises(ValueError, j.raw_decode, y, i) - - def test_bounded_int(self): - # SJ-PT-23-03, limit quadratic number parsing per Python 3.11 - max_str_digits = getattr(sys, 'get_int_max_str_digits', lambda: 4300)() - s = '1' + '0' * (max_str_digits - 1) - self.assertEqual(json.loads(s), int(s)) - self.assertRaises(ValueError, json.loads, s + '0') diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_default.py b/contrib/python/simplejson/py3/simplejson/tests/test_default.py deleted file mode 100644 index d1eacb8c4c3..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_default.py +++ /dev/null @@ -1,9 +0,0 @@ -from unittest import TestCase - -import simplejson as json - -class TestDefault(TestCase): - def test_default(self): - self.assertEqual( - json.dumps(type, default=repr), - json.dumps(repr(type))) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_dump.py b/contrib/python/simplejson/py3/simplejson/tests/test_dump.py deleted file mode 100644 index 5628489f268..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_dump.py +++ /dev/null @@ -1,255 +0,0 @@ -from unittest import TestCase -from simplejson.compat import StringIO, long_type, b, binary_type, text_type, PY3 -import simplejson as json - -class MisbehavingTextSubtype(text_type): - def __str__(self): - return "FAIL!" - -class MisbehavingBytesSubtype(binary_type): - def decode(self, encoding=None): - return "bad decode" - def __str__(self): - return "bad __str__" - def __bytes__(self): - return b("bad __bytes__") - -def as_text_type(s): - if PY3 and isinstance(s, bytes): - return s.decode('ascii') - return s - -def decode_iso_8859_15(b): - return b.decode('iso-8859-15') - -class TestDump(TestCase): - def test_dump(self): - sio = StringIO() - json.dump({}, sio) - self.assertEqual(sio.getvalue(), '{}') - - def test_constants(self): - for c in [None, True, False]: - self.assertTrue(json.loads(json.dumps(c)) is c) - self.assertTrue(json.loads(json.dumps([c]))[0] is c) - self.assertTrue(json.loads(json.dumps({'a': c}))['a'] is c) - - def test_stringify_key(self): - items = [(b('bytes'), 'bytes'), - (1.0, '1.0'), - (10, '10'), - (True, 'true'), - (False, 'false'), - (None, 'null'), - (long_type(100), '100')] - for k, expect in items: - self.assertEqual( - json.loads(json.dumps({k: expect})), - {expect: expect}) - self.assertEqual( - json.loads(json.dumps({k: expect}, sort_keys=True)), - {expect: expect}) - self.assertRaises(TypeError, json.dumps, {json: 1}) - for v in [{}, {'other': 1}, {b('derp'): 1, 'herp': 2}]: - for sort_keys in [False, True]: - v0 = dict(v) - v0[json] = 1 - v1 = dict((as_text_type(key), val) for (key, val) in v.items()) - self.assertEqual( - json.loads(json.dumps(v0, skipkeys=True, sort_keys=sort_keys)), - v1) - self.assertEqual( - json.loads(json.dumps({'': v0}, skipkeys=True, sort_keys=sort_keys)), - {'': v1}) - self.assertEqual( - json.loads(json.dumps([v0], skipkeys=True, sort_keys=sort_keys)), - [v1]) - - def test_dumps(self): - self.assertEqual(json.dumps({}), '{}') - - def test_encode_truefalse(self): - self.assertEqual(json.dumps( - {True: False, False: True}, sort_keys=True), - '{"false": true, "true": false}') - self.assertEqual( - # load first because the keys are not sorted - json.loads(json.dumps({'k1': {False: 5}, 'k2': {0: 5}})), - {'k1': {'false': 5}, 'k2': {'0': 5}}, - ) - - self.assertEqual( - json.dumps( - {2: 3.0, - 4.0: long_type(5), - False: 1, - long_type(6): True, - "7": 0}, - sort_keys=True), - '{"2": 3.0, "4.0": 5, "6": true, "7": 0, "false": 1}') - - def test_ordered_dict(self): - # http://bugs.python.org/issue6105 - items = [('one', 1), ('two', 2), ('three', 3), ('four', 4), ('five', 5)] - s = json.dumps(json.OrderedDict(items)) - self.assertEqual( - s, - '{"one": 1, "two": 2, "three": 3, "four": 4, "five": 5}') - - def test_indent_unknown_type_acceptance(self): - """ - A test against the regression mentioned at `github issue 29`_. - - The indent parameter should accept any type which pretends to be - an instance of int or long when it comes to being multiplied by - strings, even if it is not actually an int or long, for - backwards compatibility. - - .. _github issue 29: - http://github.com/simplejson/simplejson/issue/29 - """ - - class AwesomeInt(object): - """An awesome reimplementation of integers""" - - def __init__(self, *args, **kwargs): - if len(args) > 0: - # [construct from literals, objects, etc.] - # ... - - # Finally, if args[0] is an integer, store it - if isinstance(args[0], int): - self._int = args[0] - - # [various methods] - - def __mul__(self, other): - # [various ways to multiply AwesomeInt objects] - # ... finally, if the right-hand operand is not awesome enough, - # try to do a normal integer multiplication - if hasattr(self, '_int'): - return self._int * other - else: - raise NotImplementedError("To do non-awesome things with" - " this object, please construct it from an integer!") - - s = json.dumps([0, 1, 2], indent=AwesomeInt(3)) - self.assertEqual(s, '[\n 0,\n 1,\n 2\n]') - - def test_accumulator(self): - # the C API uses an accumulator that collects after 100,000 appends - lst = [0] * 100000 - self.assertEqual(json.loads(json.dumps(lst)), lst) - - def test_sort_keys(self): - # https://github.com/simplejson/simplejson/issues/106 - for num_keys in range(2, 32): - p = dict((str(x), x) for x in range(num_keys)) - sio = StringIO() - json.dump(p, sio, sort_keys=True) - self.assertEqual(sio.getvalue(), json.dumps(p, sort_keys=True)) - self.assertEqual(json.loads(sio.getvalue()), p) - - def test_misbehaving_text_subtype(self): - # https://github.com/simplejson/simplejson/issues/185 - text = "this is some text" - self.assertEqual( - json.dumps(MisbehavingTextSubtype(text)), - json.dumps(text) - ) - self.assertEqual( - json.dumps([MisbehavingTextSubtype(text)]), - json.dumps([text]) - ) - self.assertEqual( - json.dumps({MisbehavingTextSubtype(text): 42}), - json.dumps({text: 42}) - ) - - def test_misbehaving_bytes_subtype(self): - data = b("this is some data \xe2\x82\xac") - self.assertEqual( - json.dumps(MisbehavingBytesSubtype(data)), - json.dumps(data) - ) - self.assertEqual( - json.dumps([MisbehavingBytesSubtype(data)]), - json.dumps([data]) - ) - self.assertEqual( - json.dumps({MisbehavingBytesSubtype(data): 42}), - json.dumps({data: 42}) - ) - - def test_bytes_toplevel(self): - self.assertEqual(json.dumps(b('\xe2\x82\xac')), r'"\u20ac"') - self.assertRaises(UnicodeDecodeError, json.dumps, b('\xa4')) - self.assertEqual(json.dumps(b('\xa4'), encoding='iso-8859-1'), - r'"\u00a4"') - self.assertEqual(json.dumps(b('\xa4'), encoding='iso-8859-15'), - r'"\u20ac"') - if PY3: - self.assertRaises(TypeError, json.dumps, b('\xe2\x82\xac'), - encoding=None) - self.assertRaises(TypeError, json.dumps, b('\xa4'), - encoding=None) - self.assertEqual(json.dumps(b('\xa4'), encoding=None, - default=decode_iso_8859_15), - r'"\u20ac"') - else: - self.assertEqual(json.dumps(b('\xe2\x82\xac'), encoding=None), - r'"\u20ac"') - self.assertRaises(UnicodeDecodeError, json.dumps, b('\xa4'), - encoding=None) - self.assertRaises(UnicodeDecodeError, json.dumps, b('\xa4'), - encoding=None, default=decode_iso_8859_15) - - def test_bytes_nested(self): - self.assertEqual(json.dumps([b('\xe2\x82\xac')]), r'["\u20ac"]') - self.assertRaises(UnicodeDecodeError, json.dumps, [b('\xa4')]) - self.assertEqual(json.dumps([b('\xa4')], encoding='iso-8859-1'), - r'["\u00a4"]') - self.assertEqual(json.dumps([b('\xa4')], encoding='iso-8859-15'), - r'["\u20ac"]') - if PY3: - self.assertRaises(TypeError, json.dumps, [b('\xe2\x82\xac')], - encoding=None) - self.assertRaises(TypeError, json.dumps, [b('\xa4')], - encoding=None) - self.assertEqual(json.dumps([b('\xa4')], encoding=None, - default=decode_iso_8859_15), - r'["\u20ac"]') - else: - self.assertEqual(json.dumps([b('\xe2\x82\xac')], encoding=None), - r'["\u20ac"]') - self.assertRaises(UnicodeDecodeError, json.dumps, [b('\xa4')], - encoding=None) - self.assertRaises(UnicodeDecodeError, json.dumps, [b('\xa4')], - encoding=None, default=decode_iso_8859_15) - - def test_bytes_key(self): - self.assertEqual(json.dumps({b('\xe2\x82\xac'): 42}), r'{"\u20ac": 42}') - self.assertRaises(UnicodeDecodeError, json.dumps, {b('\xa4'): 42}) - self.assertEqual(json.dumps({b('\xa4'): 42}, encoding='iso-8859-1'), - r'{"\u00a4": 42}') - self.assertEqual(json.dumps({b('\xa4'): 42}, encoding='iso-8859-15'), - r'{"\u20ac": 42}') - if PY3: - self.assertRaises(TypeError, json.dumps, {b('\xe2\x82\xac'): 42}, - encoding=None) - self.assertRaises(TypeError, json.dumps, {b('\xa4'): 42}, - encoding=None) - self.assertRaises(TypeError, json.dumps, {b('\xa4'): 42}, - encoding=None, default=decode_iso_8859_15) - self.assertEqual(json.dumps({b('\xa4'): 42}, encoding=None, - skipkeys=True), - r'{}') - else: - self.assertEqual(json.dumps({b('\xe2\x82\xac'): 42}, encoding=None), - r'{"\u20ac": 42}') - self.assertRaises(UnicodeDecodeError, json.dumps, {b('\xa4'): 42}, - encoding=None) - self.assertRaises(UnicodeDecodeError, json.dumps, {b('\xa4'): 42}, - encoding=None, default=decode_iso_8859_15) - self.assertRaises(UnicodeDecodeError, json.dumps, {b('\xa4'): 42}, - encoding=None, skipkeys=True) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_encode_basestring_ascii.py b/contrib/python/simplejson/py3/simplejson/tests/test_encode_basestring_ascii.py deleted file mode 100644 index 49706bfd978..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_encode_basestring_ascii.py +++ /dev/null @@ -1,47 +0,0 @@ -from unittest import TestCase - -import simplejson.encoder -from simplejson.compat import b - -CASES = [ - (u'/\\"\ucafe\ubabe\uab98\ufcde\ubcda\uef4a\x08\x0c\n\r\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?', '"/\\\\\\"\\ucafe\\ubabe\\uab98\\ufcde\\ubcda\\uef4a\\b\\f\\n\\r\\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?"'), - (u'\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), - (u'controls', '"controls"'), - (u'\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'), - (u'{"object with 1 member":["array with 1 element"]}', '"{\\"object with 1 member\\":[\\"array with 1 element\\"]}"'), - (u' s p a c e d ', '" s p a c e d "'), - (u'\U0001d120', '"\\ud834\\udd20"'), - (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'), - (b('\xce\xb1\xce\xa9'), '"\\u03b1\\u03a9"'), - (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'), - (b('\xce\xb1\xce\xa9'), '"\\u03b1\\u03a9"'), - (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'), - (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'), - (u"`1~!@#$%^&*()_+-={':[,]}|;.</>?", '"`1~!@#$%^&*()_+-={\':[,]}|;.</>?"'), - (u'\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'), - (u'\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), -] - -class TestEncodeBaseStringAscii(TestCase): - def test_py_encode_basestring_ascii(self): - self._test_encode_basestring_ascii(simplejson.encoder.py_encode_basestring_ascii) - - def test_c_encode_basestring_ascii(self): - if not simplejson.encoder.c_encode_basestring_ascii: - return - self._test_encode_basestring_ascii(simplejson.encoder.c_encode_basestring_ascii) - - def _test_encode_basestring_ascii(self, encode_basestring_ascii): - fname = encode_basestring_ascii.__name__ - for input_string, expect in CASES: - result = encode_basestring_ascii(input_string) - #self.assertEqual(result, expect, - # '{0!r} != {1!r} for {2}({3!r})'.format( - # result, expect, fname, input_string)) - self.assertEqual(result, expect, - '%r != %r for %s(%r)' % (result, expect, fname, input_string)) - - def test_sorted_dict(self): - items = [('one', 1), ('two', 2), ('three', 3), ('four', 4), ('five', 5)] - s = simplejson.dumps(dict(items), sort_keys=True) - self.assertEqual(s, '{"five": 5, "four": 4, "one": 1, "three": 3, "two": 2}') diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_encode_for_html.py b/contrib/python/simplejson/py3/simplejson/tests/test_encode_for_html.py deleted file mode 100644 index 3a840aa48cb..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_encode_for_html.py +++ /dev/null @@ -1,38 +0,0 @@ -import unittest - -import simplejson as json - -class TestEncodeForHTML(unittest.TestCase): - - def setUp(self): - self.decoder = json.JSONDecoder() - self.encoder = json.JSONEncoderForHTML() - self.non_ascii_encoder = json.JSONEncoderForHTML(ensure_ascii=False) - - def test_basic_encode(self): - self.assertEqual(r'"\u0026"', self.encoder.encode('&')) - self.assertEqual(r'"\u003c"', self.encoder.encode('<')) - self.assertEqual(r'"\u003e"', self.encoder.encode('>')) - self.assertEqual(r'"\u2028"', self.encoder.encode(u'\u2028')) - - def test_non_ascii_basic_encode(self): - self.assertEqual(r'"\u0026"', self.non_ascii_encoder.encode('&')) - self.assertEqual(r'"\u003c"', self.non_ascii_encoder.encode('<')) - self.assertEqual(r'"\u003e"', self.non_ascii_encoder.encode('>')) - self.assertEqual(r'"\u2028"', self.non_ascii_encoder.encode(u'\u2028')) - - def test_basic_roundtrip(self): - for char in '&<>': - self.assertEqual( - char, self.decoder.decode( - self.encoder.encode(char))) - - def test_prevent_script_breakout(self): - bad_string = '</script><script>alert("gotcha")</script>' - self.assertEqual( - r'"\u003c/script\u003e\u003cscript\u003e' - r'alert(\"gotcha\")\u003c/script\u003e"', - self.encoder.encode(bad_string)) - self.assertEqual( - bad_string, self.decoder.decode( - self.encoder.encode(bad_string))) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_errors.py b/contrib/python/simplejson/py3/simplejson/tests/test_errors.py deleted file mode 100644 index d5738254541..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_errors.py +++ /dev/null @@ -1,68 +0,0 @@ -import sys, pickle -from unittest import TestCase - -import simplejson as json -from simplejson.compat import text_type, b - -class TestErrors(TestCase): - def test_string_keys_error(self): - data = [{'a': 'A', 'b': (2, 4), 'c': 3.0, ('d',): 'D tuple'}] - try: - json.dumps(data) - except TypeError: - err = sys.exc_info()[1] - else: - self.fail('Expected TypeError') - self.assertEqual(str(err), - 'keys must be str, int, float, bool or None, not tuple') - - def test_not_serializable(self): - try: - json.dumps(json) - except TypeError: - err = sys.exc_info()[1] - else: - self.fail('Expected TypeError') - self.assertEqual(str(err), - 'Object of type module is not JSON serializable') - - def test_decode_error(self): - err = None - try: - json.loads('{}\na\nb') - except json.JSONDecodeError: - err = sys.exc_info()[1] - else: - self.fail('Expected JSONDecodeError') - self.assertEqual(err.lineno, 2) - self.assertEqual(err.colno, 1) - self.assertEqual(err.endlineno, 3) - self.assertEqual(err.endcolno, 2) - - def test_scan_error(self): - err = None - for t in (text_type, b): - try: - json.loads(t('{"asdf": "')) - except json.JSONDecodeError: - err = sys.exc_info()[1] - else: - self.fail('Expected JSONDecodeError') - self.assertEqual(err.lineno, 1) - self.assertEqual(err.colno, 10) - - def test_error_is_pickable(self): - err = None - try: - json.loads('{}\na\nb') - except json.JSONDecodeError: - err = sys.exc_info()[1] - else: - self.fail('Expected JSONDecodeError') - s = pickle.dumps(err) - e = pickle.loads(s) - - self.assertEqual(err.msg, e.msg) - self.assertEqual(err.doc, e.doc) - self.assertEqual(err.pos, e.pos) - self.assertEqual(err.end, e.end) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_fail.py b/contrib/python/simplejson/py3/simplejson/tests/test_fail.py deleted file mode 100644 index 5f9a8f698cf..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_fail.py +++ /dev/null @@ -1,178 +0,0 @@ -import sys -from unittest import TestCase - -import simplejson as json - -# 2007-10-05 -JSONDOCS = [ - # http://json.org/JSON_checker/test/fail1.json - '"A JSON payload should be an object or array, not a string."', - # http://json.org/JSON_checker/test/fail2.json - '["Unclosed array"', - # http://json.org/JSON_checker/test/fail3.json - '{unquoted_key: "keys must be quoted"}', - # http://json.org/JSON_checker/test/fail4.json - '["extra comma",]', - # http://json.org/JSON_checker/test/fail5.json - '["double extra comma",,]', - # http://json.org/JSON_checker/test/fail6.json - '[ , "<-- missing value"]', - # http://json.org/JSON_checker/test/fail7.json - '["Comma after the close"],', - # http://json.org/JSON_checker/test/fail8.json - '["Extra close"]]', - # http://json.org/JSON_checker/test/fail9.json - '{"Extra comma": true,}', - # http://json.org/JSON_checker/test/fail10.json - '{"Extra value after close": true} "misplaced quoted value"', - # http://json.org/JSON_checker/test/fail11.json - '{"Illegal expression": 1 + 2}', - # http://json.org/JSON_checker/test/fail12.json - '{"Illegal invocation": alert()}', - # http://json.org/JSON_checker/test/fail13.json - '{"Numbers cannot have leading zeroes": 013}', - # http://json.org/JSON_checker/test/fail14.json - '{"Numbers cannot be hex": 0x14}', - # http://json.org/JSON_checker/test/fail15.json - '["Illegal backslash escape: \\x15"]', - # http://json.org/JSON_checker/test/fail16.json - '[\\naked]', - # http://json.org/JSON_checker/test/fail17.json - '["Illegal backslash escape: \\017"]', - # http://json.org/JSON_checker/test/fail18.json - '[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]', - # http://json.org/JSON_checker/test/fail19.json - '{"Missing colon" null}', - # http://json.org/JSON_checker/test/fail20.json - '{"Double colon":: null}', - # http://json.org/JSON_checker/test/fail21.json - '{"Comma instead of colon", null}', - # http://json.org/JSON_checker/test/fail22.json - '["Colon instead of comma": false]', - # http://json.org/JSON_checker/test/fail23.json - '["Bad value", truth]', - # http://json.org/JSON_checker/test/fail24.json - "['single quote']", - # http://json.org/JSON_checker/test/fail25.json - '["\ttab\tcharacter\tin\tstring\t"]', - # http://json.org/JSON_checker/test/fail26.json - '["tab\\ character\\ in\\ string\\ "]', - # http://json.org/JSON_checker/test/fail27.json - '["line\nbreak"]', - # http://json.org/JSON_checker/test/fail28.json - '["line\\\nbreak"]', - # http://json.org/JSON_checker/test/fail29.json - '[0e]', - # http://json.org/JSON_checker/test/fail30.json - '[0e+]', - # http://json.org/JSON_checker/test/fail31.json - '[0e+-1]', - # http://json.org/JSON_checker/test/fail32.json - '{"Comma instead if closing brace": true,', - # http://json.org/JSON_checker/test/fail33.json - '["mismatch"}', - # http://code.google.com/p/simplejson/issues/detail?id=3 - u'["A\u001FZ control characters in string"]', - # misc based on coverage - '{', - '{]', - '{"foo": "bar"]', - '{"foo": "bar"', - 'nul', - 'nulx', - '-', - '-x', - '-e', - '-e0', - '-Infinite', - '-Inf', - 'Infinit', - 'Infinite', - 'NaM', - 'NuN', - 'falsy', - 'fal', - 'trug', - 'tru', - '1e', - '1ex', - '1e-', - '1e-x', -] - -SKIPS = { - 1: "why not have a string payload?", - 18: "spec doesn't specify any nesting limitations", -} - -class TestFail(TestCase): - def test_failures(self): - for idx, doc in enumerate(JSONDOCS): - idx = idx + 1 - if idx in SKIPS: - json.loads(doc) - continue - try: - json.loads(doc) - except json.JSONDecodeError: - pass - else: - self.fail("Expected failure for fail%d.json: %r" % (idx, doc)) - - def test_array_decoder_issue46(self): - # http://code.google.com/p/simplejson/issues/detail?id=46 - for doc in [u'[,]', '[,]']: - try: - json.loads(doc) - except json.JSONDecodeError: - e = sys.exc_info()[1] - self.assertEqual(e.pos, 1) - self.assertEqual(e.lineno, 1) - self.assertEqual(e.colno, 2) - except Exception: - e = sys.exc_info()[1] - self.fail("Unexpected exception raised %r %s" % (e, e)) - else: - self.fail("Unexpected success parsing '[,]'") - - def test_truncated_input(self): - test_cases = [ - ('', 'Expecting value', 0), - ('[', "Expecting value or ']'", 1), - ('[42', "Expecting ',' delimiter", 3), - ('[42,', 'Expecting value', 4), - ('["', 'Unterminated string starting at', 1), - ('["spam', 'Unterminated string starting at', 1), - ('["spam"', "Expecting ',' delimiter", 7), - ('["spam",', 'Expecting value', 8), - ('{', "Expecting property name enclosed in double quotes or '}'", 1), - ('{"', 'Unterminated string starting at', 1), - ('{"spam', 'Unterminated string starting at', 1), - ('{"spam"', "Expecting ':' delimiter", 7), - ('{"spam":', 'Expecting value', 8), - ('{"spam":42', "Expecting ',' delimiter", 10), - ('{"spam":42,', 'Expecting property name enclosed in double quotes', - 11), - ('"', 'Unterminated string starting at', 0), - ('"spam', 'Unterminated string starting at', 0), - ('[,', "Expecting value", 1), - ('--', 'Expecting value', 0), - ('"\x18d', "Invalid control character %r", 1), - ] - for data, msg, idx in test_cases: - try: - json.loads(data) - except json.JSONDecodeError: - e = sys.exc_info()[1] - self.assertEqual( - e.msg[:len(msg)], - msg, - "%r doesn't start with %r for %r" % (e.msg, msg, data)) - self.assertEqual( - e.pos, idx, - "pos %r != %r for %r" % (e.pos, idx, data)) - except Exception: - e = sys.exc_info()[1] - self.fail("Unexpected exception raised %r %s" % (e, e)) - else: - self.fail("Unexpected success parsing '%r'" % (data,)) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_float.py b/contrib/python/simplejson/py3/simplejson/tests/test_float.py deleted file mode 100644 index fed1bd97cd6..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_float.py +++ /dev/null @@ -1,46 +0,0 @@ -import sys -import math -from unittest import TestCase -from simplejson.compat import long_type, text_type -import simplejson as json -from simplejson.decoder import NaN, PosInf, NegInf - -class TestFloat(TestCase): - def test_degenerates_allow(self): - for inf in (PosInf, NegInf): - self.assertEqual(json.loads(json.dumps(inf, allow_nan=True), allow_nan=True), inf) - # Python 2.5 doesn't have math.isnan - nan = json.loads(json.dumps(NaN, allow_nan=True), allow_nan=True) - self.assertTrue((0 + nan) != nan) - - def test_degenerates_ignore(self): - for f in (PosInf, NegInf, NaN): - self.assertEqual(json.loads(json.dumps(f, ignore_nan=True)), None) - - def test_degenerates_deny(self): - for f in (PosInf, NegInf, NaN): - self.assertRaises(ValueError, json.dumps, f, allow_nan=False) - for s in ('Infinity', '-Infinity', 'NaN'): - self.assertRaises(ValueError, json.loads, s, allow_nan=False) - self.assertRaises(ValueError, json.loads, s) - - def test_floats(self): - for num in [1617161771.7650001, math.pi, math.pi**100, - math.pi**-100, 3.1]: - self.assertEqual(float(json.dumps(num)), num) - self.assertEqual(json.loads(json.dumps(num)), num) - self.assertEqual(json.loads(text_type(json.dumps(num))), num) - - def test_ints(self): - for num in [1, long_type(1), 1<<32, 1<<64]: - self.assertEqual(json.dumps(num), str(num)) - self.assertEqual(int(json.dumps(num)), num) - self.assertEqual(json.loads(json.dumps(num)), num) - self.assertEqual(json.loads(text_type(json.dumps(num))), num) - - def test_float_range(self): - try: - float_range = [sys.float_info.min, sys.float_info.max] - except AttributeError: - float_range = [2.2250738585072014e-308, 1.7976931348623157e+308] - self.assertEqual(json.loads(json.dumps(float_range)), float_range) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_for_json.py b/contrib/python/simplejson/py3/simplejson/tests/test_for_json.py deleted file mode 100644 index b791b883b04..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_for_json.py +++ /dev/null @@ -1,97 +0,0 @@ -import unittest -import simplejson as json - - -class ForJson(object): - def for_json(self): - return {'for_json': 1} - - -class NestedForJson(object): - def for_json(self): - return {'nested': ForJson()} - - -class ForJsonList(object): - def for_json(self): - return ['list'] - - -class DictForJson(dict): - def for_json(self): - return {'alpha': 1} - - -class ListForJson(list): - def for_json(self): - return ['list'] - - -class TestForJson(unittest.TestCase): - def assertRoundTrip(self, obj, other, for_json=True): - if for_json is None: - # None will use the default - s = json.dumps(obj) - else: - s = json.dumps(obj, for_json=for_json) - self.assertEqual( - json.loads(s), - other) - - def test_for_json_encodes_stand_alone_object(self): - self.assertRoundTrip( - ForJson(), - ForJson().for_json()) - - def test_for_json_encodes_object_nested_in_dict(self): - self.assertRoundTrip( - {'hooray': ForJson()}, - {'hooray': ForJson().for_json()}) - - def test_for_json_encodes_object_nested_in_list_within_dict(self): - self.assertRoundTrip( - {'list': [0, ForJson(), 2, 3]}, - {'list': [0, ForJson().for_json(), 2, 3]}) - - def test_for_json_encodes_object_nested_within_object(self): - self.assertRoundTrip( - NestedForJson(), - {'nested': {'for_json': 1}}) - - def test_for_json_encodes_list(self): - self.assertRoundTrip( - ForJsonList(), - ForJsonList().for_json()) - - def test_for_json_encodes_list_within_object(self): - self.assertRoundTrip( - {'nested': ForJsonList()}, - {'nested': ForJsonList().for_json()}) - - def test_for_json_encodes_dict_subclass(self): - self.assertRoundTrip( - DictForJson(a=1), - DictForJson(a=1).for_json()) - - def test_for_json_encodes_list_subclass(self): - self.assertRoundTrip( - ListForJson(['l']), - ListForJson(['l']).for_json()) - - def test_for_json_ignored_if_not_true_with_dict_subclass(self): - for for_json in (None, False): - self.assertRoundTrip( - DictForJson(a=1), - {'a': 1}, - for_json=for_json) - - def test_for_json_ignored_if_not_true_with_list_subclass(self): - for for_json in (None, False): - self.assertRoundTrip( - ListForJson(['l']), - ['l'], - for_json=for_json) - - def test_raises_typeerror_if_for_json_not_true_with_object(self): - self.assertRaises(TypeError, json.dumps, ForJson()) - self.assertRaises(TypeError, json.dumps, ForJson(), for_json=False) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_indent.py b/contrib/python/simplejson/py3/simplejson/tests/test_indent.py deleted file mode 100644 index cea25a575ed..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_indent.py +++ /dev/null @@ -1,86 +0,0 @@ -from unittest import TestCase -import textwrap - -import simplejson as json -from simplejson.compat import StringIO - -class TestIndent(TestCase): - def test_indent(self): - h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', - 'i-vhbjkhnth', - {'nifty': 87}, {'field': 'yes', 'morefield': False} ] - - expect = textwrap.dedent("""\ - [ - \t[ - \t\t"blorpie" - \t], - \t[ - \t\t"whoops" - \t], - \t[], - \t"d-shtaeou", - \t"d-nthiouh", - \t"i-vhbjkhnth", - \t{ - \t\t"nifty": 87 - \t}, - \t{ - \t\t"field": "yes", - \t\t"morefield": false - \t} - ]""") - - - d1 = json.dumps(h) - d2 = json.dumps(h, indent='\t', sort_keys=True, separators=(',', ': ')) - d3 = json.dumps(h, indent=' ', sort_keys=True, separators=(',', ': ')) - d4 = json.dumps(h, indent=2, sort_keys=True, separators=(',', ': ')) - - h1 = json.loads(d1) - h2 = json.loads(d2) - h3 = json.loads(d3) - h4 = json.loads(d4) - - self.assertEqual(h1, h) - self.assertEqual(h2, h) - self.assertEqual(h3, h) - self.assertEqual(h4, h) - self.assertEqual(d3, expect.replace('\t', ' ')) - self.assertEqual(d4, expect.replace('\t', ' ')) - # NOTE: Python 2.4 textwrap.dedent converts tabs to spaces, - # so the following is expected to fail. Python 2.4 is not a - # supported platform in simplejson 2.1.0+. - self.assertEqual(d2, expect) - - def test_indent0(self): - h = {3: 1} - def check(indent, expected): - d1 = json.dumps(h, indent=indent) - self.assertEqual(d1, expected) - - sio = StringIO() - json.dump(h, sio, indent=indent) - self.assertEqual(sio.getvalue(), expected) - - # indent=0 should emit newlines - check(0, '{\n"3": 1\n}') - # indent=None is more compact - check(None, '{"3": 1}') - - def test_separators(self): - lst = [1,2,3,4] - expect = '[\n1,\n2,\n3,\n4\n]' - expect_spaces = '[\n1, \n2, \n3, \n4\n]' - # Ensure that separators still works - self.assertEqual( - expect_spaces, - json.dumps(lst, indent=0, separators=(', ', ': '))) - # Force the new defaults - self.assertEqual( - expect, - json.dumps(lst, indent=0, separators=(',', ': '))) - # Added in 2.1.4 - self.assertEqual( - expect, - json.dumps(lst, indent=0)) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_item_sort_key.py b/contrib/python/simplejson/py3/simplejson/tests/test_item_sort_key.py deleted file mode 100644 index 98971b86ac7..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_item_sort_key.py +++ /dev/null @@ -1,27 +0,0 @@ -from unittest import TestCase - -import simplejson as json -from operator import itemgetter - -class TestItemSortKey(TestCase): - def test_simple_first(self): - a = {'a': 1, 'c': 5, 'jack': 'jill', 'pick': 'axe', 'array': [1, 5, 6, 9], 'tuple': (83, 12, 3), 'crate': 'dog', 'zeak': 'oh'} - self.assertEqual( - '{"a": 1, "c": 5, "crate": "dog", "jack": "jill", "pick": "axe", "zeak": "oh", "array": [1, 5, 6, 9], "tuple": [83, 12, 3]}', - json.dumps(a, item_sort_key=json.simple_first)) - - def test_case(self): - a = {'a': 1, 'c': 5, 'Jack': 'jill', 'pick': 'axe', 'Array': [1, 5, 6, 9], 'tuple': (83, 12, 3), 'crate': 'dog', 'zeak': 'oh'} - self.assertEqual( - '{"Array": [1, 5, 6, 9], "Jack": "jill", "a": 1, "c": 5, "crate": "dog", "pick": "axe", "tuple": [83, 12, 3], "zeak": "oh"}', - json.dumps(a, item_sort_key=itemgetter(0))) - self.assertEqual( - '{"a": 1, "Array": [1, 5, 6, 9], "c": 5, "crate": "dog", "Jack": "jill", "pick": "axe", "tuple": [83, 12, 3], "zeak": "oh"}', - json.dumps(a, item_sort_key=lambda kv: kv[0].lower())) - - def test_item_sort_key_value(self): - # https://github.com/simplejson/simplejson/issues/173 - a = {'a': 1, 'b': 0} - self.assertEqual( - '{"b": 0, "a": 1}', - json.dumps(a, item_sort_key=lambda kv: kv[1])) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_iterable.py b/contrib/python/simplejson/py3/simplejson/tests/test_iterable.py deleted file mode 100644 index 35d3e759111..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_iterable.py +++ /dev/null @@ -1,31 +0,0 @@ -import unittest -from simplejson.compat import StringIO - -import simplejson as json - -def iter_dumps(obj, **kw): - return ''.join(json.JSONEncoder(**kw).iterencode(obj)) - -def sio_dump(obj, **kw): - sio = StringIO() - json.dumps(obj, **kw) - return sio.getvalue() - -class TestIterable(unittest.TestCase): - def test_iterable(self): - for l in ([], [1], [1, 2], [1, 2, 3]): - for opts in [{}, {'indent': 2}]: - for dumps in (json.dumps, iter_dumps, sio_dump): - expect = dumps(l, **opts) - default_expect = dumps(sum(l), **opts) - # Default is False - self.assertRaises(TypeError, dumps, iter(l), **opts) - self.assertRaises(TypeError, dumps, iter(l), iterable_as_array=False, **opts) - self.assertEqual(expect, dumps(iter(l), iterable_as_array=True, **opts)) - # Ensure that the "default" gets called - self.assertEqual(default_expect, dumps(iter(l), default=sum, **opts)) - self.assertEqual(default_expect, dumps(iter(l), iterable_as_array=False, default=sum, **opts)) - # Ensure that the "default" does not get called - self.assertEqual( - expect, - dumps(iter(l), iterable_as_array=True, default=sum, **opts)) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_namedtuple.py b/contrib/python/simplejson/py3/simplejson/tests/test_namedtuple.py deleted file mode 100644 index cc0f8aa1d7d..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_namedtuple.py +++ /dev/null @@ -1,174 +0,0 @@ -from __future__ import absolute_import -import unittest -import simplejson as json -from simplejson.compat import StringIO - -try: - from unittest import mock -except ImportError: - mock = None - -try: - from collections import namedtuple -except ImportError: - class Value(tuple): - def __new__(cls, *args): - return tuple.__new__(cls, args) - - def _asdict(self): - return {'value': self[0]} - class Point(tuple): - def __new__(cls, *args): - return tuple.__new__(cls, args) - - def _asdict(self): - return {'x': self[0], 'y': self[1]} -else: - Value = namedtuple('Value', ['value']) - Point = namedtuple('Point', ['x', 'y']) - -class DuckValue(object): - def __init__(self, *args): - self.value = Value(*args) - - def _asdict(self): - return self.value._asdict() - -class DuckPoint(object): - def __init__(self, *args): - self.point = Point(*args) - - def _asdict(self): - return self.point._asdict() - -class DeadDuck(object): - _asdict = None - -class DeadDict(dict): - _asdict = None - -CONSTRUCTORS = [ - lambda v: v, - lambda v: [v], - lambda v: [{'key': v}], -] - -class TestNamedTuple(unittest.TestCase): - def test_namedtuple_dumps(self): - for v in [Value(1), Point(1, 2), DuckValue(1), DuckPoint(1, 2)]: - d = v._asdict() - self.assertEqual(d, json.loads(json.dumps(v))) - self.assertEqual( - d, - json.loads(json.dumps(v, namedtuple_as_object=True))) - self.assertEqual(d, json.loads(json.dumps(v, tuple_as_array=False))) - self.assertEqual( - d, - json.loads(json.dumps(v, namedtuple_as_object=True, - tuple_as_array=False))) - - def test_namedtuple_dumps_false(self): - for v in [Value(1), Point(1, 2)]: - l = list(v) - self.assertEqual( - l, - json.loads(json.dumps(v, namedtuple_as_object=False))) - self.assertRaises(TypeError, json.dumps, v, - tuple_as_array=False, namedtuple_as_object=False) - - def test_namedtuple_dump(self): - for v in [Value(1), Point(1, 2), DuckValue(1), DuckPoint(1, 2)]: - d = v._asdict() - sio = StringIO() - json.dump(v, sio) - self.assertEqual(d, json.loads(sio.getvalue())) - sio = StringIO() - json.dump(v, sio, namedtuple_as_object=True) - self.assertEqual( - d, - json.loads(sio.getvalue())) - sio = StringIO() - json.dump(v, sio, tuple_as_array=False) - self.assertEqual(d, json.loads(sio.getvalue())) - sio = StringIO() - json.dump(v, sio, namedtuple_as_object=True, - tuple_as_array=False) - self.assertEqual( - d, - json.loads(sio.getvalue())) - - def test_namedtuple_dump_false(self): - for v in [Value(1), Point(1, 2)]: - l = list(v) - sio = StringIO() - json.dump(v, sio, namedtuple_as_object=False) - self.assertEqual( - l, - json.loads(sio.getvalue())) - self.assertRaises(TypeError, json.dump, v, StringIO(), - tuple_as_array=False, namedtuple_as_object=False) - - def test_asdict_not_callable_dump(self): - for f in CONSTRUCTORS: - self.assertRaises( - TypeError, - json.dump, - f(DeadDuck()), - StringIO(), - namedtuple_as_object=True - ) - sio = StringIO() - json.dump(f(DeadDict()), sio, namedtuple_as_object=True) - self.assertEqual( - json.dumps(f({})), - sio.getvalue()) - self.assertRaises( - TypeError, - json.dump, - f(Value), - StringIO(), - namedtuple_as_object=True - ) - - def test_asdict_not_callable_dumps(self): - for f in CONSTRUCTORS: - self.assertRaises(TypeError, - json.dumps, f(DeadDuck()), namedtuple_as_object=True) - self.assertRaises( - TypeError, - json.dumps, - f(Value), - namedtuple_as_object=True - ) - self.assertEqual( - json.dumps(f({})), - json.dumps(f(DeadDict()), namedtuple_as_object=True)) - - def test_asdict_unbound_method_dumps(self): - for f in CONSTRUCTORS: - self.assertEqual( - json.dumps(f(Value), default=lambda v: v.__name__), - json.dumps(f(Value.__name__)) - ) - - def test_asdict_does_not_return_dict(self): - if not mock: - if hasattr(unittest, "SkipTest"): - raise unittest.SkipTest("unittest.mock required") - else: - print("unittest.mock not available") - return - fake = mock.Mock() - self.assertTrue(hasattr(fake, '_asdict')) - self.assertTrue(callable(fake._asdict)) - self.assertFalse(isinstance(fake._asdict(), dict)) - # https://github.com/simplejson/simplejson/pull/284 - # when running under a debug build of CPython (COPTS=-UNDEBUG) - # a C assertion could fire due to an unchecked error of an PyDict - # API call on a non-dict internally in _speedups.c. Without a debug - # build of CPython this test likely passes either way despite the - # potential for internal data corruption. Getting it to crash in - # a debug build is not always easy either as it requires an - # assert(!PyErr_Occurred()) that could fire later on. - with self.assertRaises(TypeError): - json.dumps({23: fake}, namedtuple_as_object=True, for_json=False) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_pass1.py b/contrib/python/simplejson/py3/simplejson/tests/test_pass1.py deleted file mode 100644 index f0b5b10e761..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_pass1.py +++ /dev/null @@ -1,71 +0,0 @@ -from unittest import TestCase - -import simplejson as json - -# from http://json.org/JSON_checker/test/pass1.json -JSON = r''' -[ - "JSON Test Pattern pass1", - {"object with 1 member":["array with 1 element"]}, - {}, - [], - -42, - true, - false, - null, - { - "integer": 1234567890, - "real": -9876.543210, - "e": 0.123456789e-12, - "E": 1.234567890E+34, - "": 23456789012E66, - "zero": 0, - "one": 1, - "space": " ", - "quote": "\"", - "backslash": "\\", - "controls": "\b\f\n\r\t", - "slash": "/ & \/", - "alpha": "abcdefghijklmnopqrstuvwyz", - "ALPHA": "ABCDEFGHIJKLMNOPQRSTUVWYZ", - "digit": "0123456789", - "special": "`1~!@#$%^&*()_+-={':[,]}|;.</>?", - "hex": "\u0123\u4567\u89AB\uCDEF\uabcd\uef4A", - "true": true, - "false": false, - "null": null, - "array":[ ], - "object":{ }, - "address": "50 St. James Street", - "url": "http://www.JSON.org/", - "comment": "// /* <!-- --", - "# -- --> */": " ", - " s p a c e d " :[1,2 , 3 - -, - -4 , 5 , 6 ,7 ],"compact": [1,2,3,4,5,6,7], - "jsontext": "{\"object with 1 member\":[\"array with 1 element\"]}", - "quotes": "" \u0022 %22 0x22 034 "", - "\/\\\"\uCAFE\uBABE\uAB98\uFCDE\ubcda\uef4A\b\f\n\r\t`1~!@#$%^&*()_+-=[]{}|;:',./<>?" -: "A key can be any string" - }, - 0.5 ,98.6 -, -99.44 -, - -1066, -1e1, -0.1e1, -1e-1, -1e00,2e+00,2e-00 -,"rosebud"] -''' - -class TestPass1(TestCase): - def test_parse(self): - # test in/out equivalence and parsing - res = json.loads(JSON) - out = json.dumps(res) - self.assertEqual(res, json.loads(out)) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_pass2.py b/contrib/python/simplejson/py3/simplejson/tests/test_pass2.py deleted file mode 100644 index 5d812b3bbe2..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_pass2.py +++ /dev/null @@ -1,14 +0,0 @@ -from unittest import TestCase -import simplejson as json - -# from http://json.org/JSON_checker/test/pass2.json -JSON = r''' -[[[[[[[[[[[[[[[[[[["Not too deep"]]]]]]]]]]]]]]]]]]] -''' - -class TestPass2(TestCase): - def test_parse(self): - # test in/out equivalence and parsing - res = json.loads(JSON) - out = json.dumps(res) - self.assertEqual(res, json.loads(out)) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_pass3.py b/contrib/python/simplejson/py3/simplejson/tests/test_pass3.py deleted file mode 100644 index 821d60b22cc..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_pass3.py +++ /dev/null @@ -1,20 +0,0 @@ -from unittest import TestCase - -import simplejson as json - -# from http://json.org/JSON_checker/test/pass3.json -JSON = r''' -{ - "JSON Test Pattern pass3": { - "The outermost value": "must be an object or array.", - "In this test": "It is an object." - } -} -''' - -class TestPass3(TestCase): - def test_parse(self): - # test in/out equivalence and parsing - res = json.loads(JSON) - out = json.dumps(res) - self.assertEqual(res, json.loads(out)) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_raw_json.py b/contrib/python/simplejson/py3/simplejson/tests/test_raw_json.py deleted file mode 100644 index 1dfcc2c5f95..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_raw_json.py +++ /dev/null @@ -1,47 +0,0 @@ -import unittest -import simplejson as json - -dct1 = { - 'key1': 'value1' -} - -dct2 = { - 'key2': 'value2', - 'd1': dct1 -} - -dct3 = { - 'key2': 'value2', - 'd1': json.dumps(dct1) -} - -dct4 = { - 'key2': 'value2', - 'd1': json.RawJSON(json.dumps(dct1)) -} - - -class TestRawJson(unittest.TestCase): - - def test_normal_str(self): - self.assertNotEqual(json.dumps(dct2), json.dumps(dct3)) - - def test_raw_json_str(self): - self.assertEqual(json.dumps(dct2), json.dumps(dct4)) - self.assertEqual(dct2, json.loads(json.dumps(dct4))) - - def test_list(self): - self.assertEqual( - json.dumps([dct2]), - json.dumps([json.RawJSON(json.dumps(dct2))])) - self.assertEqual( - [dct2], - json.loads(json.dumps([json.RawJSON(json.dumps(dct2))]))) - - def test_direct(self): - self.assertEqual( - json.dumps(dct2), - json.dumps(json.RawJSON(json.dumps(dct2)))) - self.assertEqual( - dct2, - json.loads(json.dumps(json.RawJSON(json.dumps(dct2))))) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_recursion.py b/contrib/python/simplejson/py3/simplejson/tests/test_recursion.py deleted file mode 100644 index 662eb667ece..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_recursion.py +++ /dev/null @@ -1,67 +0,0 @@ -from unittest import TestCase - -import simplejson as json - -class JSONTestObject: - pass - - -class RecursiveJSONEncoder(json.JSONEncoder): - recurse = False - def default(self, o): - if o is JSONTestObject: - if self.recurse: - return [JSONTestObject] - else: - return 'JSONTestObject' - return json.JSONEncoder.default(o) - - -class TestRecursion(TestCase): - def test_listrecursion(self): - x = [] - x.append(x) - try: - json.dumps(x) - except ValueError: - pass - else: - self.fail("didn't raise ValueError on list recursion") - x = [] - y = [x] - x.append(y) - try: - json.dumps(x) - except ValueError: - pass - else: - self.fail("didn't raise ValueError on alternating list recursion") - y = [] - x = [y, y] - # ensure that the marker is cleared - json.dumps(x) - - def test_dictrecursion(self): - x = {} - x["test"] = x - try: - json.dumps(x) - except ValueError: - pass - else: - self.fail("didn't raise ValueError on dict recursion") - x = {} - y = {"a": x, "b": x} - # ensure that the marker is cleared - json.dumps(y) - - def test_defaultrecursion(self): - enc = RecursiveJSONEncoder() - self.assertEqual(enc.encode(JSONTestObject), '"JSONTestObject"') - enc.recurse = True - try: - enc.encode(JSONTestObject) - except ValueError: - pass - else: - self.fail("didn't raise ValueError on default recursion") diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_scanstring.py b/contrib/python/simplejson/py3/simplejson/tests/test_scanstring.py deleted file mode 100644 index 1f544834209..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_scanstring.py +++ /dev/null @@ -1,200 +0,0 @@ -import sys -from unittest import TestCase - -import simplejson as json -import simplejson.decoder -from simplejson.compat import b, PY3 - -class TestScanString(TestCase): - # The bytes type is intentionally not used in most of these tests - # under Python 3 because the decoder immediately coerces to str before - # calling scanstring. In Python 2 we are testing the code paths - # for both unicode and str. - # - # The reason this is done is because Python 3 would require - # entirely different code paths for parsing bytes and str. - # - def test_py_scanstring(self): - self._test_scanstring(simplejson.decoder.py_scanstring) - - def test_c_scanstring(self): - if not simplejson.decoder.c_scanstring: - return - self._test_scanstring(simplejson.decoder.c_scanstring) - - self.assertTrue(isinstance(simplejson.decoder.c_scanstring('""', 0)[0], str)) - - def _test_scanstring(self, scanstring): - if sys.maxunicode == 65535: - self.assertEqual( - scanstring(u'"z\U0001d120x"', 1, None, True), - (u'z\U0001d120x', 6)) - else: - self.assertEqual( - scanstring(u'"z\U0001d120x"', 1, None, True), - (u'z\U0001d120x', 5)) - - self.assertEqual( - scanstring('"\\u007b"', 1, None, True), - (u'{', 8)) - - self.assertEqual( - scanstring('"A JSON payload should be an object or array, not a string."', 1, None, True), - (u'A JSON payload should be an object or array, not a string.', 60)) - - self.assertEqual( - scanstring('["Unclosed array"', 2, None, True), - (u'Unclosed array', 17)) - - self.assertEqual( - scanstring('["extra comma",]', 2, None, True), - (u'extra comma', 14)) - - self.assertEqual( - scanstring('["double extra comma",,]', 2, None, True), - (u'double extra comma', 21)) - - self.assertEqual( - scanstring('["Comma after the close"],', 2, None, True), - (u'Comma after the close', 24)) - - self.assertEqual( - scanstring('["Extra close"]]', 2, None, True), - (u'Extra close', 14)) - - self.assertEqual( - scanstring('{"Extra comma": true,}', 2, None, True), - (u'Extra comma', 14)) - - self.assertEqual( - scanstring('{"Extra value after close": true} "misplaced quoted value"', 2, None, True), - (u'Extra value after close', 26)) - - self.assertEqual( - scanstring('{"Illegal expression": 1 + 2}', 2, None, True), - (u'Illegal expression', 21)) - - self.assertEqual( - scanstring('{"Illegal invocation": alert()}', 2, None, True), - (u'Illegal invocation', 21)) - - self.assertEqual( - scanstring('{"Numbers cannot have leading zeroes": 013}', 2, None, True), - (u'Numbers cannot have leading zeroes', 37)) - - self.assertEqual( - scanstring('{"Numbers cannot be hex": 0x14}', 2, None, True), - (u'Numbers cannot be hex', 24)) - - self.assertEqual( - scanstring('[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]', 21, None, True), - (u'Too deep', 30)) - - self.assertEqual( - scanstring('{"Missing colon" null}', 2, None, True), - (u'Missing colon', 16)) - - self.assertEqual( - scanstring('{"Double colon":: null}', 2, None, True), - (u'Double colon', 15)) - - self.assertEqual( - scanstring('{"Comma instead of colon", null}', 2, None, True), - (u'Comma instead of colon', 25)) - - self.assertEqual( - scanstring('["Colon instead of comma": false]', 2, None, True), - (u'Colon instead of comma', 25)) - - self.assertEqual( - scanstring('["Bad value", truth]', 2, None, True), - (u'Bad value', 12)) - - for c in map(chr, range(0x00, 0x1f)): - self.assertEqual( - scanstring(c + '"', 0, None, False), - (c, 2)) - self.assertRaises( - ValueError, - scanstring, c + '"', 0, None, True) - - self.assertRaises(ValueError, scanstring, '', 0, None, True) - self.assertRaises(ValueError, scanstring, 'a', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\u', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\u0', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\u01', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\u012', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\u0123', 0, None, True) - if sys.maxunicode > 65535: - self.assertRaises(ValueError, - scanstring, '\\ud834\\u"', 0, None, True) - self.assertRaises(ValueError, - scanstring, '\\ud834\\x0123"', 0, None, True) - - self.assertRaises(json.JSONDecodeError, scanstring, '\\u-123"', 0, None, True) - # SJ-PT-23-01: Invalid Handling of Broken Unicode Escape Sequences - self.assertRaises(json.JSONDecodeError, scanstring, '\\u EDD"', 0, None, True) - - def test_issue3623(self): - self.assertRaises(ValueError, json.decoder.scanstring, "xxx", 1, - "xxx") - self.assertRaises(UnicodeDecodeError, - json.encoder.encode_basestring_ascii, b("xx\xff")) - - def test_overflow(self): - # Python 2.5 does not have maxsize, Python 3 does not have maxint - maxsize = getattr(sys, 'maxsize', getattr(sys, 'maxint', None)) - assert maxsize is not None - self.assertRaises(OverflowError, json.decoder.scanstring, "xxx", - maxsize + 1) - - def test_surrogates(self): - scanstring = json.decoder.scanstring - - def assertScan(given, expect, test_utf8=True): - givens = [given] - if not PY3 and test_utf8: - givens.append(given.encode('utf8')) - for given in givens: - (res, count) = scanstring(given, 1, None, True) - self.assertEqual(len(given), count) - self.assertEqual(res, expect) - - assertScan( - u'"z\\ud834\\u0079x"', - u'z\ud834yx') - assertScan( - u'"z\\ud834\\udd20x"', - u'z\U0001d120x') - assertScan( - u'"z\\ud834\\ud834\\udd20x"', - u'z\ud834\U0001d120x') - assertScan( - u'"z\\ud834x"', - u'z\ud834x') - assertScan( - u'"z\\udd20x"', - u'z\udd20x') - assertScan( - u'"z\ud834x"', - u'z\ud834x') - # It may look strange to join strings together, but Python is drunk. - # https://gist.github.com/etrepum/5538443 - assertScan( - u'"z\\ud834\udd20x12345"', - u''.join([u'z\ud834', u'\udd20x12345'])) - assertScan( - u'"z\ud834\\udd20x"', - u''.join([u'z\ud834', u'\udd20x'])) - # these have different behavior given UTF8 input, because the surrogate - # pair may be joined (in maxunicode > 65535 builds) - assertScan( - u''.join([u'"z\ud834', u'\udd20x"']), - u''.join([u'z\ud834', u'\udd20x']), - test_utf8=False) - - self.assertRaises(ValueError, - scanstring, u'"z\\ud83x"', 1, None, True) - self.assertRaises(ValueError, - scanstring, u'"z\\ud834\\udd2x"', 1, None, True) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_separators.py b/contrib/python/simplejson/py3/simplejson/tests/test_separators.py deleted file mode 100644 index 91b4d4fb6f3..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_separators.py +++ /dev/null @@ -1,42 +0,0 @@ -import textwrap -from unittest import TestCase - -import simplejson as json - - -class TestSeparators(TestCase): - def test_separators(self): - h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', 'i-vhbjkhnth', - {'nifty': 87}, {'field': 'yes', 'morefield': False} ] - - expect = textwrap.dedent("""\ - [ - [ - "blorpie" - ] , - [ - "whoops" - ] , - [] , - "d-shtaeou" , - "d-nthiouh" , - "i-vhbjkhnth" , - { - "nifty" : 87 - } , - { - "field" : "yes" , - "morefield" : false - } - ]""") - - - d1 = json.dumps(h) - d2 = json.dumps(h, indent=' ', sort_keys=True, separators=(' ,', ' : ')) - - h1 = json.loads(d1) - h2 = json.loads(d2) - - self.assertEqual(h1, h) - self.assertEqual(h2, h) - self.assertEqual(d2, expect) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_speedups.py b/contrib/python/simplejson/py3/simplejson/tests/test_speedups.py deleted file mode 100644 index 8b146df9a05..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_speedups.py +++ /dev/null @@ -1,114 +0,0 @@ -from __future__ import with_statement - -import sys -import unittest -from unittest import TestCase - -import simplejson -from simplejson import encoder, decoder, scanner -from simplejson.compat import PY3, long_type, b - - -def has_speedups(): - return encoder.c_make_encoder is not None - - -def skip_if_speedups_missing(func): - def wrapper(*args, **kwargs): - if not has_speedups(): - if hasattr(unittest, 'SkipTest'): - raise unittest.SkipTest("C Extension not available") - else: - sys.stdout.write("C Extension not available") - return - return func(*args, **kwargs) - - return wrapper - - -class BadBool: - def __bool__(self): - 1/0 - __nonzero__ = __bool__ - - -class TestDecode(TestCase): - @skip_if_speedups_missing - def test_make_scanner(self): - self.assertRaises(AttributeError, scanner.c_make_scanner, 1) - - @skip_if_speedups_missing - def test_bad_bool_args(self): - def test(value): - decoder.JSONDecoder(strict=BadBool()).decode(value) - self.assertRaises(ZeroDivisionError, test, '""') - self.assertRaises(ZeroDivisionError, test, '{}') - if not PY3: - self.assertRaises(ZeroDivisionError, test, u'""') - self.assertRaises(ZeroDivisionError, test, u'{}') - -class TestEncode(TestCase): - @skip_if_speedups_missing - def test_make_encoder(self): - self.assertRaises( - TypeError, - encoder.c_make_encoder, - None, - ("\xCD\x7D\x3D\x4E\x12\x4C\xF9\x79\xD7" - "\x52\xBA\x82\xF2\x27\x4A\x7D\xA0\xCA\x75"), - None - ) - - @skip_if_speedups_missing - def test_bad_str_encoder(self): - # Issue #31505: There shouldn't be an assertion failure in case - # c_make_encoder() receives a bad encoder() argument. - import decimal - def bad_encoder1(*args): - return None - enc = encoder.c_make_encoder( - None, lambda obj: str(obj), - bad_encoder1, None, ': ', ', ', - False, False, False, {}, False, False, False, - None, None, 'utf-8', False, False, decimal.Decimal, False) - self.assertRaises(TypeError, enc, 'spam', 4) - self.assertRaises(TypeError, enc, {'spam': 42}, 4) - - def bad_encoder2(*args): - 1/0 - enc = encoder.c_make_encoder( - None, lambda obj: str(obj), - bad_encoder2, None, ': ', ', ', - False, False, False, {}, False, False, False, - None, None, 'utf-8', False, False, decimal.Decimal, False) - self.assertRaises(ZeroDivisionError, enc, 'spam', 4) - - @skip_if_speedups_missing - def test_bad_bool_args(self): - def test(name): - encoder.JSONEncoder(**{name: BadBool()}).encode({}) - self.assertRaises(ZeroDivisionError, test, 'skipkeys') - self.assertRaises(ZeroDivisionError, test, 'ensure_ascii') - self.assertRaises(ZeroDivisionError, test, 'check_circular') - self.assertRaises(ZeroDivisionError, test, 'allow_nan') - self.assertRaises(ZeroDivisionError, test, 'sort_keys') - self.assertRaises(ZeroDivisionError, test, 'use_decimal') - self.assertRaises(ZeroDivisionError, test, 'namedtuple_as_object') - self.assertRaises(ZeroDivisionError, test, 'tuple_as_array') - self.assertRaises(ZeroDivisionError, test, 'bigint_as_string') - self.assertRaises(ZeroDivisionError, test, 'for_json') - self.assertRaises(ZeroDivisionError, test, 'ignore_nan') - self.assertRaises(ZeroDivisionError, test, 'iterable_as_array') - - @skip_if_speedups_missing - def test_int_as_string_bitcount_overflow(self): - long_count = long_type(2)**32+31 - def test(): - encoder.JSONEncoder(int_as_string_bitcount=long_count).encode(0) - self.assertRaises((TypeError, OverflowError), test) - - if PY3: - @skip_if_speedups_missing - def test_bad_encoding(self): - with self.assertRaises(UnicodeEncodeError): - encoder.JSONEncoder(encoding='\udcff').encode({b('key'): 123}) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_str_subclass.py b/contrib/python/simplejson/py3/simplejson/tests/test_str_subclass.py deleted file mode 100644 index b6c8351f264..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_str_subclass.py +++ /dev/null @@ -1,21 +0,0 @@ -from unittest import TestCase - -import simplejson -from simplejson.compat import text_type - -# Tests for issue demonstrated in https://github.com/simplejson/simplejson/issues/144 -class WonkyTextSubclass(text_type): - def __getslice__(self, start, end): - return self.__class__('not what you wanted!') - -class TestStrSubclass(TestCase): - def test_dump_load(self): - for s in ['', '"hello"', 'text', u'\u005c']: - self.assertEqual( - s, - simplejson.loads(simplejson.dumps(WonkyTextSubclass(s)))) - - self.assertEqual( - s, - simplejson.loads(simplejson.dumps(WonkyTextSubclass(s), - ensure_ascii=False))) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_subclass.py b/contrib/python/simplejson/py3/simplejson/tests/test_subclass.py deleted file mode 100644 index 2bae3b6309e..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_subclass.py +++ /dev/null @@ -1,37 +0,0 @@ -from unittest import TestCase -import simplejson as json - -from decimal import Decimal - -class AlternateInt(int): - def __repr__(self): - return 'invalid json' - __str__ = __repr__ - - -class AlternateFloat(float): - def __repr__(self): - return 'invalid json' - __str__ = __repr__ - - -# class AlternateDecimal(Decimal): -# def __repr__(self): -# return 'invalid json' - - -class TestSubclass(TestCase): - def test_int(self): - self.assertEqual(json.dumps(AlternateInt(1)), '1') - self.assertEqual(json.dumps(AlternateInt(-1)), '-1') - self.assertEqual(json.loads(json.dumps({AlternateInt(1): 1})), {'1': 1}) - - def test_float(self): - self.assertEqual(json.dumps(AlternateFloat(1.0)), '1.0') - self.assertEqual(json.dumps(AlternateFloat(-1.0)), '-1.0') - self.assertEqual(json.loads(json.dumps({AlternateFloat(1.0): 1})), {'1.0': 1}) - - # NOTE: Decimal subclasses are not supported as-is - # def test_decimal(self): - # self.assertEqual(json.dumps(AlternateDecimal('1.0')), '1.0') - # self.assertEqual(json.dumps(AlternateDecimal('-1.0')), '-1.0') diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_tool.py b/contrib/python/simplejson/py3/simplejson/tests/test_tool.py deleted file mode 100644 index 914bff8bca5..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_tool.py +++ /dev/null @@ -1,114 +0,0 @@ -from __future__ import with_statement -import os -import sys -import textwrap -import unittest -import subprocess -import tempfile -try: - # Python 3.x - from test.support import strip_python_stderr -except ImportError: - # Python 2.6+ - try: - from test.test_support import strip_python_stderr - except ImportError: - # Python 2.5 - import re - def strip_python_stderr(stderr): - return re.sub( - r"\[\d+ refs\]\r?\n?$".encode(), - "".encode(), - stderr).strip() - -def open_temp_file(): - if sys.version_info >= (2, 6): - file = tempfile.NamedTemporaryFile(delete=False) - filename = file.name - else: - fd, filename = tempfile.mkstemp() - file = os.fdopen(fd, 'w+b') - return file, filename - -class TestTool(unittest.TestCase): - data = """ - - [["blorpie"],[ "whoops" ] , [ - ],\t"d-shtaeou",\r"d-nthiouh", - "i-vhbjkhnth", {"nifty":87}, {"morefield" :\tfalse,"field" - :"yes"} ] - """ - - expect = textwrap.dedent("""\ - [ - [ - "blorpie" - ], - [ - "whoops" - ], - [], - "d-shtaeou", - "d-nthiouh", - "i-vhbjkhnth", - { - "nifty": 87 - }, - { - "field": "yes", - "morefield": false - } - ] - """) - - def runTool(self, args=None, data=None): - argv = [sys.executable, '-m', 'simplejson.tool'] - if args: - argv.extend(args) - proc = subprocess.Popen(argv, - stdin=subprocess.PIPE, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE) - out, err = proc.communicate(data) - self.assertEqual(strip_python_stderr(err), ''.encode()) - self.assertEqual(proc.returncode, 0) - return out.decode('utf8').splitlines() - - def test_stdin_stdout(self): - self.assertEqual( - self.runTool(data=self.data.encode()), - self.expect.splitlines()) - - def test_infile_stdout(self): - infile, infile_name = open_temp_file() - try: - infile.write(self.data.encode()) - infile.close() - self.assertEqual( - self.runTool(args=[infile_name]), - self.expect.splitlines()) - finally: - os.unlink(infile_name) - - def test_infile_outfile(self): - infile, infile_name = open_temp_file() - try: - infile.write(self.data.encode()) - infile.close() - # outfile will get overwritten by tool, so the delete - # may not work on some platforms. Do it manually. - outfile, outfile_name = open_temp_file() - try: - outfile.close() - self.assertEqual( - self.runTool(args=[infile_name, outfile_name]), - []) - with open(outfile_name, 'rb') as f: - self.assertEqual( - f.read().decode('utf8').splitlines(), - self.expect.splitlines() - ) - finally: - os.unlink(outfile_name) - finally: - os.unlink(infile_name) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_tuple.py b/contrib/python/simplejson/py3/simplejson/tests/test_tuple.py deleted file mode 100644 index 4ad7b0e861b..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_tuple.py +++ /dev/null @@ -1,47 +0,0 @@ -import unittest - -from simplejson.compat import StringIO -import simplejson as json - -class TestTuples(unittest.TestCase): - def test_tuple_array_dumps(self): - t = (1, 2, 3) - expect = json.dumps(list(t)) - # Default is True - self.assertEqual(expect, json.dumps(t)) - self.assertEqual(expect, json.dumps(t, tuple_as_array=True)) - self.assertRaises(TypeError, json.dumps, t, tuple_as_array=False) - # Ensure that the "default" does not get called - self.assertEqual(expect, json.dumps(t, default=repr)) - self.assertEqual(expect, json.dumps(t, tuple_as_array=True, - default=repr)) - # Ensure that the "default" gets called - self.assertEqual( - json.dumps(repr(t)), - json.dumps(t, tuple_as_array=False, default=repr)) - - def test_tuple_array_dump(self): - t = (1, 2, 3) - expect = json.dumps(list(t)) - # Default is True - sio = StringIO() - json.dump(t, sio) - self.assertEqual(expect, sio.getvalue()) - sio = StringIO() - json.dump(t, sio, tuple_as_array=True) - self.assertEqual(expect, sio.getvalue()) - self.assertRaises(TypeError, json.dump, t, StringIO(), - tuple_as_array=False) - # Ensure that the "default" does not get called - sio = StringIO() - json.dump(t, sio, default=repr) - self.assertEqual(expect, sio.getvalue()) - sio = StringIO() - json.dump(t, sio, tuple_as_array=True, default=repr) - self.assertEqual(expect, sio.getvalue()) - # Ensure that the "default" gets called - sio = StringIO() - json.dump(t, sio, tuple_as_array=False, default=repr) - self.assertEqual( - json.dumps(repr(t)), - sio.getvalue()) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_unicode.py b/contrib/python/simplejson/py3/simplejson/tests/test_unicode.py deleted file mode 100644 index 0c7b1a6c444..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tests/test_unicode.py +++ /dev/null @@ -1,154 +0,0 @@ -import sys -import codecs -from unittest import TestCase - -import simplejson as json -from simplejson.compat import unichr, text_type, b, BytesIO - -class TestUnicode(TestCase): - def test_encoding1(self): - encoder = json.JSONEncoder(encoding='utf-8') - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - s = u.encode('utf-8') - ju = encoder.encode(u) - js = encoder.encode(s) - self.assertEqual(ju, js) - - def test_encoding2(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - s = u.encode('utf-8') - ju = json.dumps(u, encoding='utf-8') - js = json.dumps(s, encoding='utf-8') - self.assertEqual(ju, js) - - def test_encoding3(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps(u) - self.assertEqual(j, '"\\u03b1\\u03a9"') - - def test_encoding4(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps([u]) - self.assertEqual(j, '["\\u03b1\\u03a9"]') - - def test_encoding5(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps(u, ensure_ascii=False) - self.assertEqual(j, u'"' + u + u'"') - - def test_encoding6(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps([u], ensure_ascii=False) - self.assertEqual(j, u'["' + u + u'"]') - - def test_big_unicode_encode(self): - u = u'\U0001d120' - self.assertEqual(json.dumps(u), '"\\ud834\\udd20"') - self.assertEqual(json.dumps(u, ensure_ascii=False), u'"\U0001d120"') - - def test_big_unicode_decode(self): - u = u'z\U0001d120x' - self.assertEqual(json.loads('"' + u + '"'), u) - self.assertEqual(json.loads('"z\\ud834\\udd20x"'), u) - - def test_unicode_decode(self): - for i in range(0, 0xd7ff): - u = unichr(i) - #s = '"\\u{0:04x}"'.format(i) - s = '"\\u%04x"' % (i,) - self.assertEqual(json.loads(s), u) - - def test_object_pairs_hook_with_unicode(self): - s = u'{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}' - p = [(u"xkd", 1), (u"kcw", 2), (u"art", 3), (u"hxm", 4), - (u"qrt", 5), (u"pad", 6), (u"hoy", 7)] - self.assertEqual(json.loads(s), eval(s)) - self.assertEqual(json.loads(s, object_pairs_hook=lambda x: x), p) - od = json.loads(s, object_pairs_hook=json.OrderedDict) - self.assertEqual(od, json.OrderedDict(p)) - self.assertEqual(type(od), json.OrderedDict) - # the object_pairs_hook takes priority over the object_hook - self.assertEqual(json.loads(s, - object_pairs_hook=json.OrderedDict, - object_hook=lambda x: None), - json.OrderedDict(p)) - - - def test_default_encoding(self): - self.assertEqual(json.loads(u'{"a": "\xe9"}'.encode('utf-8')), - {'a': u'\xe9'}) - - def test_unicode_preservation(self): - self.assertEqual(type(json.loads(u'""')), text_type) - self.assertEqual(type(json.loads(u'"a"')), text_type) - self.assertEqual(type(json.loads(u'["a"]')[0]), text_type) - - def test_ensure_ascii_false_returns_unicode(self): - # http://code.google.com/p/simplejson/issues/detail?id=48 - self.assertEqual(type(json.dumps([], ensure_ascii=False)), text_type) - self.assertEqual(type(json.dumps(0, ensure_ascii=False)), text_type) - self.assertEqual(type(json.dumps({}, ensure_ascii=False)), text_type) - self.assertEqual(type(json.dumps("", ensure_ascii=False)), text_type) - - def test_ensure_ascii_false_bytestring_encoding(self): - # http://code.google.com/p/simplejson/issues/detail?id=48 - doc1 = {u'quux': b('Arr\xc3\xaat sur images')} - doc2 = {u'quux': u'Arr\xeat sur images'} - doc_ascii = '{"quux": "Arr\\u00eat sur images"}' - doc_unicode = u'{"quux": "Arr\xeat sur images"}' - self.assertEqual(json.dumps(doc1), doc_ascii) - self.assertEqual(json.dumps(doc2), doc_ascii) - self.assertEqual(json.dumps(doc1, ensure_ascii=False), doc_unicode) - self.assertEqual(json.dumps(doc2, ensure_ascii=False), doc_unicode) - - def test_ensure_ascii_linebreak_encoding(self): - # http://timelessrepo.com/json-isnt-a-javascript-subset - s1 = u'\u2029\u2028' - s2 = s1.encode('utf8') - expect = '"\\u2029\\u2028"' - expect_non_ascii = u'"\u2029\u2028"' - self.assertEqual(json.dumps(s1), expect) - self.assertEqual(json.dumps(s2), expect) - self.assertEqual(json.dumps(s1, ensure_ascii=False), expect_non_ascii) - self.assertEqual(json.dumps(s2, ensure_ascii=False), expect_non_ascii) - - def test_invalid_escape_sequences(self): - # incomplete escape sequence - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u1') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u12') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u123') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u1234') - # invalid escape sequence - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u123x"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u12x4"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u1x34"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ux234"') - if sys.maxunicode > 65535: - # invalid escape sequence for low surrogate - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u0"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u00"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u000"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u000x"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u00x0"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u0x00"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\ux000"') - - def test_ensure_ascii_still_works(self): - # in the ascii range, ensure that everything is the same - for c in map(unichr, range(0, 127)): - self.assertEqual( - json.dumps(c, ensure_ascii=False), - json.dumps(c)) - snowman = u'\N{SNOWMAN}' - self.assertEqual( - json.dumps(c, ensure_ascii=False), - '"' + c + '"') - - def test_strip_bom(self): - content = u"\u3053\u3093\u306b\u3061\u308f" - json_doc = codecs.BOM_UTF8 + b(json.dumps(content)) - self.assertEqual(json.load(BytesIO(json_doc)), content) - for doc in json_doc, json_doc.decode('utf8'): - self.assertEqual(json.loads(doc), content) diff --git a/contrib/python/simplejson/py3/simplejson/tool.py b/contrib/python/simplejson/py3/simplejson/tool.py deleted file mode 100644 index 062e8e2c181..00000000000 --- a/contrib/python/simplejson/py3/simplejson/tool.py +++ /dev/null @@ -1,42 +0,0 @@ -r"""Command-line tool to validate and pretty-print JSON - -Usage:: - - $ echo '{"json":"obj"}' | python -m simplejson.tool - { - "json": "obj" - } - $ echo '{ 1.2:3.4}' | python -m simplejson.tool - Expecting property name: line 1 column 2 (char 2) - -""" -from __future__ import with_statement -import sys -import simplejson as json - -def main(): - if len(sys.argv) == 1: - infile = sys.stdin - outfile = sys.stdout - elif len(sys.argv) == 2: - infile = open(sys.argv[1], 'r') - outfile = sys.stdout - elif len(sys.argv) == 3: - infile = open(sys.argv[1], 'r') - outfile = open(sys.argv[2], 'w') - else: - raise SystemExit(sys.argv[0] + " [infile [outfile]]") - with infile: - try: - obj = json.load(infile, - object_pairs_hook=json.OrderedDict, - use_decimal=True) - except ValueError: - raise SystemExit(sys.exc_info()[1]) - with outfile: - json.dump(obj, outfile, sort_keys=True, indent=' ', use_decimal=True) - outfile.write('\n') - - -if __name__ == '__main__': - main() diff --git a/contrib/python/simplejson/py3/tests/ya.make b/contrib/python/simplejson/py3/tests/ya.make deleted file mode 100644 index 3f788396a7c..00000000000 --- a/contrib/python/simplejson/py3/tests/ya.make +++ /dev/null @@ -1,47 +0,0 @@ -PY3TEST() - -PEERDIR( - contrib/python/simplejson -) - -SRCDIR( - contrib/python/simplejson/py3/simplejson/tests -) - -TEST_SRCS( - __init__.py - test_bigint_as_string.py - test_bitsize_int_as_string.py - test_check_circular.py - test_decimal.py - test_decode.py - test_default.py - test_dump.py - test_encode_basestring_ascii.py - test_encode_for_html.py - test_errors.py - test_fail.py - test_float.py - test_for_json.py - test_indent.py - test_item_sort_key.py - test_iterable.py - test_namedtuple.py - test_pass1.py - test_pass2.py - test_pass3.py - test_raw_json.py - test_recursion.py - test_scanstring.py - test_separators.py - test_speedups.py - test_str_subclass.py - test_subclass.py - # test_tool.py - test_tuple.py - test_unicode.py -) - -NO_LINT() - -END() diff --git a/contrib/python/simplejson/py3/ya.make b/contrib/python/simplejson/py3/ya.make deleted file mode 100644 index 85e677db4bd..00000000000 --- a/contrib/python/simplejson/py3/ya.make +++ /dev/null @@ -1,48 +0,0 @@ -# Generated by devtools/yamaker (pypi). - -PY3_LIBRARY() - -VERSION(3.20.2) - -LICENSE(MIT) - -NO_COMPILER_WARNINGS() - -NO_LINT() - -NO_CHECK_IMPORTS( - simplejson.ordered_dict -) - -SRCS( - simplejson/_speedups.c -) - -PY_REGISTER( - simplejson._speedups -) - -PY_SRCS( - TOP_LEVEL - simplejson/__init__.py - simplejson/compat.py - simplejson/decoder.py - simplejson/encoder.py - simplejson/errors.py - simplejson/ordered_dict.py - simplejson/raw_json.py - simplejson/scanner.py - simplejson/tool.py -) - -RESOURCE_FILES( - PREFIX contrib/python/simplejson/py3/ - .dist-info/METADATA - .dist-info/top_level.txt -) - -END() - -RECURSE_FOR_TESTS( - tests -) diff --git a/contrib/python/simplejson/ya.make b/contrib/python/simplejson/ya.make deleted file mode 100644 index 8ba230decd8..00000000000 --- a/contrib/python/simplejson/ya.make +++ /dev/null @@ -1,20 +0,0 @@ -PY23_LIBRARY() - -LICENSE(Service-Py23-Proxy) - -VERSION(Service-proxy-version) - -IF (PYTHON2) - PEERDIR(contrib/python/simplejson/py2) -ELSE() - PEERDIR(contrib/python/simplejson/py3) -ENDIF() - -NO_LINT() - -END() - -RECURSE( - py2 - py3 -) diff --git a/contrib/python/toml/py3/ya.make b/contrib/python/toml/py3/ya.make index be97015fd78..9d29cc702f1 100644 --- a/contrib/python/toml/py3/ya.make +++ b/contrib/python/toml/py3/ya.make @@ -2,6 +2,8 @@ PY3_LIBRARY() +SUBSCRIBER(g:python-contrib) + VERSION(0.10.2) LICENSE(MIT) diff --git a/library/python/prctl/prctl.pyx b/library/python/prctl/prctl.pyx deleted file mode 100644 index 707a79c22f1..00000000000 --- a/library/python/prctl/prctl.pyx +++ /dev/null @@ -1,14 +0,0 @@ -cdef extern from "<sys/prctl.h>": - int prctl(int option, unsigned long arg2, unsigned long arg3, unsigned long arg4, unsigned long arg5); - - -PR_SET_PDEATHSIG = 1 -PR_SET_CHILD_SUBREAPER = 36 - - -def set_pdeathsig(signum): - return prctl(PR_SET_PDEATHSIG, signum, 0, 0, 0) - - -def set_child_subreaper(val): - return prctl(PR_SET_CHILD_SUBREAPER, val, 0, 0, 0) diff --git a/library/python/prctl/ya.make b/library/python/prctl/ya.make deleted file mode 100644 index 163626e91db..00000000000 --- a/library/python/prctl/ya.make +++ /dev/null @@ -1,9 +0,0 @@ -PY23_LIBRARY() - -PY_SRCS( - prctl.pyx -) - -BUILD_ONLY_IF(WARNING LINUX) - -END() diff --git a/library/python/protobuf/runtime/ya.make b/library/python/protobuf/runtime/ya.make deleted file mode 100644 index 0bb52fffe63..00000000000 --- a/library/python/protobuf/runtime/ya.make +++ /dev/null @@ -1,13 +0,0 @@ -PY23_LIBRARY() - -IF (USE_VANILLA_PROTOC AND NOT PYTHON2) - PEERDIR( - contrib/python/protobuf_std - ) -ELSE() - PEERDIR( - contrib/python/protobuf - ) -ENDIF() - -END() diff --git a/library/python/testing/swag/daemon.py b/library/python/testing/swag/daemon.py deleted file mode 100644 index 81fd0eb97b1..00000000000 --- a/library/python/testing/swag/daemon.py +++ /dev/null @@ -1,294 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import logging -import os -import signal -import tempfile -import shutil - -import six - -try: - from . import gdb -except ValueError: - import gdb - -from yatest.common import process, output_path, TimeoutError, cores - -MAX_IO_LEN = 1024 * 10 -GYGABYTES = 1 << 30 - -logger = logging.getLogger(__name__) - - -def run_daemon(command, check_exit_code=True, shell=False, timeout=5, cwd=None, - env=None, stdin=None, stdout=None, stderr=None, creationflags=0): - daemon = Daemon(command, check_exit_code, shell, timeout, cwd, env, stdin, stdout, stderr, creationflags) - daemon.run() - return daemon - - -def get_free_space(path): - stats = os.statvfs(path) - return stats.f_bavail * stats.f_frsize - - -class DaemonError(RuntimeError): - def __init__(self, message, stdout=None, stderr=None, exit_code=None): - lst = [ - "Daemon failed with message: {message}.".format(message=message), - ] - if exit_code is not None: - lst.append( - "Process exit_code = {exit_code}.".format(exit_code=exit_code) - ) - if stdout is not None: - lst.append( - "Stdout: {stdout}".format(stdout=stdout) - ) - if stderr is not None: - lst.append( - "Stderr: {stderr}".format(stderr=stderr) - ) - - super(DaemonError, self).__init__('\n'.join(lst)) - - -class Daemon(object): - def __init__(self, command, check_exit_code=True, shell=False, timeout=5, cwd=None, - env=None, stdin=None, stdout=None, stderr=None, creationflags=0): - if cwd is None: - cwd = tempfile.mkdtemp() - self.cwd = cwd - - self.stdoutf = stdout or tempfile.NamedTemporaryFile(dir=self.cwd, prefix="stdout_", delete=False) - self.stderrf = stderr or tempfile.NamedTemporaryFile(dir=self.cwd, prefix="stderr_", delete=False) - self.stdinf = stdin or tempfile.NamedTemporaryFile(dir=self.cwd, prefix="stdin_", delete=False) - - self.cmd = command - if isinstance(command, six.string_types): - self.cmd = [arg for arg in command.split() if arg] - self.daemon = None - self.name = os.path.basename(self.cmd[0]) - - self._shell = shell - self._env = env - self._creationflags = creationflags - - self._check_exit_code = check_exit_code - self._timeout = timeout - - def before_start(self): - pass - - def after_start(self): - pass - - def before_stop(self): - pass - - def after_stop(self): - pass - - def is_alive(self): - return self.daemon and self.daemon.running - - def required_args(self): - return [] - - def check_run(self): - """This function checks that daemon is running. By default it - checks only the process status. But you can override it to - check your binary specific marks like 'port is busy' and - others.""" - return self.is_alive() - - def run(self): - if self.check_run(): - logger.error("Can't run %s.\nProcess already started" % self.cmd) - raise DaemonError("daemon already started.") - - try: - self.before_start() - except Exception: - logger.exception("Exception in user hook before_start") - self.daemon = process.execute(self.cmd[:1] + self.required_args() + self.cmd[1:], - False, - shell=self._shell, - cwd=self.cwd, - env=self._env, - stdin=self.stdinf, - stdout=self.stdoutf, - stderr=self.stderrf, - creationflags=self._creationflags, - wait=False) - stdout, stderr = self.__communicate() - timeout_reason_msg = "Failed to execute '{cmd}'.\n\tstdout: {out}\n\tstderr: {err}".format( - cmd=" ".join(self.cmd), - out=stdout, - err=stderr) - try: - process.wait_for(self.check_run, self._timeout, timeout_reason_msg, sleep_time=0.1) - except process.TimeoutError: - self.raise_on_death(timeout_reason_msg) - - if not self.is_alive(): - self.raise_on_death("WHY? %s %s" % (self.daemon, self.daemon.running)) - - try: - self.after_start() - except Exception as e: - msg = "Exception in user hook after_start. Exception: %s" % str(e) - logger.exception(msg) - - return self - - def raise_on_death(self, additional_text=""): - stdout = "[NO STDOUT]" - stderr = "[NO STDERR]" - - if self.stdoutf and self.stdinf: - stdout, stderr = self.__communicate() - if self.daemon and getattr(self.daemon, "process"): - self.check_coredump() - - raise DaemonError( - Daemon.__log_failed( - "process {} unexpectedly finished. \n\n {}".format(self.cmd, additional_text), - stdout, - stderr - ) - ) - - def check_coredump(self): - try: - core_file = cores.recover_core_dump_file(self.cmd[0], self.cwd, self.daemon.process.pid) - if core_file: - logger.debug(core_file + " found, maybe this is our coredump file") - self.save_coredump(core_file) - else: - logger.debug("Core dump file was not found") - except Exception as e: - logger.warn("While checking coredump: " + str(e)) - - def save_coredump(self, core_file): - output_core_dir = output_path("cores") - shared_core_file = os.path.join(output_core_dir, os.path.basename(core_file)) - if not os.path.isdir(output_core_dir): - os.mkdir(output_core_dir) - - short_bt, _ = gdb.dump_traceback(executable=self.cmd[0], core_file=core_file, - output_file=shared_core_file + ".trace.txt") - if short_bt: - logger.error("Short backtrace = \n" + "=" * 80 + "\n" + short_bt + "\n" + "=" * 80) - - space_left = float(get_free_space(output_core_dir)) - if space_left > 5 * GYGABYTES: - shutil.copy2( - core_file, - shared_core_file - ) - os.chmod(shared_core_file, 0o755) - logger.debug("Saved to " + output_core_dir) - - else: - logger.error("Not enough space left on device (%s GB). Won't save %s file" % (float(space_left / GYGABYTES), core_file)) - - def stop(self, kill=False): - if not self.is_alive() and self.daemon.exit_code == 0: - return - - if not self.is_alive(): - stdout, stderr = self.__communicate() - self.check_coredump() - try: - self.after_stop() - except Exception: - logger.exception("Exception in user hook after_stop.") - - raise DaemonError( - Daemon.__log_failed( - "process {} unexpectedly finished with exit code {}.".format(self.cmd, self.daemon.exit_code), - stdout, - stderr - ), - exit_code=self.daemon.exit_code - ) - - try: - self.before_stop() - except Exception: - logger.exception("Exception in user hook before_stop.") - - stderr, stdout = self.__communicate() - timeout_reason_msg = "Cannot stop {cmd}.\n\tstdout: {out}\n\tstderr: {err}".format( - cmd=" ".join(self.cmd), - out=stdout, - err=stderr) - if not kill: - self.daemon.process.send_signal(signal.SIGINT) - try: # soft wait for. trying to kill with sigint - process.wait_for(lambda: not self.is_alive(), self._timeout, timeout_reason_msg, sleep_time=0.1) - except TimeoutError: - pass - - is_killed = False - if self.is_alive(): - self.daemon.process.send_signal(signal.SIGKILL) - is_killed = True - - process.wait_for(lambda: not self.is_alive(), self._timeout, timeout_reason_msg, sleep_time=0.1) - - try: - self.after_stop() - except Exception: - logger.exception("Exception in user hook after_stop") - - if self.daemon.running: - stdout, stderr = self.__communicate() - msg = "cannot stop daemon {cmd}\n\tstdout: {out}\n\tstderr: {err}".format( - cmd=' '.join(self.cmd), - out=stdout, - err=stderr - ) - logger.error(msg) - raise DaemonError(msg, stdout=stdout, stderr=stderr, exit_code=self.daemon.exit_code) - - stdout, stderr = self.__communicate() - logger.debug( - "Process stopped: {cmd}.\n\tstdout:\n{out}\n\tstderr:\n{err}".format( - cmd=" ".join(self.cmd), - out=stdout, - err=stderr - ) - ) - if not is_killed: - self.check_coredump() - if self._check_exit_code and self.daemon.exit_code != 0: - stdout, stderr = self.__communicate() - raise DaemonError("Bad exit_code.", stdout=stdout, stderr=stderr, exit_code=self.daemon.exit_code) - else: - logger.warning("Exit code is not checked, cos binary was stopped by sigkill") - - def _read_io(self, file_obj): - file_obj.flush() - - cur_pos = file_obj.tell() - seek_pos_from_end = max(-cur_pos, -MAX_IO_LEN) - file_obj.seek(seek_pos_from_end, os.SEEK_END) - return file_obj.read() - - def __communicate(self): - stderr = self._read_io(self.stderrf) - stdout = self._read_io(self.stdoutf) - return stdout, stderr - - @staticmethod - def __log_failed(msg, stderr, stdout): - final_msg = '{msg}\nstdout: {out}\nstderr: {err}'.format( - msg=msg, - out=stdout, - err=stderr) - logger.error(msg) - return final_msg diff --git a/library/python/testing/swag/gdb.py b/library/python/testing/swag/gdb.py deleted file mode 100644 index b9a3694d906..00000000000 --- a/library/python/testing/swag/gdb.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import logging - -import yatest.common - -logger = logging.getLogger(__name__) - -SHORT_DUMP_COMMAND = "{gdb} {executable} {core_file} --eval-command='backtrace full' --batch -q" -LONG_DUMP_COMMAND = "{gdb} {executable} {core_file} --eval-command='thread apply all bt full' --batch -q" - - -def get_gdb(): - return yatest.common.gdb_path() - - -def run_gdb_command(command, stdout_file, stderr_file): - logger.debug("Running gdb command %s" % command) - - with open(stdout_file, "w") as out, open(stderr_file, "w") as err: - yatest.common.process.execute( - command, - check_exit_code=True, - wait=True, - shell=True, - stdout=out, - stderr=err - ) - - -def dump_traceback(executable, core_file, output_file): - """ - Dumps traceback if its possible - - :param executable: binary for gdb - :param core_file: core file for gdb - :param output_file: file to dump traceback to, also dump full traceback to <output_file + ".full"> - - :return: string tuple (short_backtrace, full_backtrace) - """ - try: - gdb = get_gdb() - short_dump_command = SHORT_DUMP_COMMAND.format(gdb=gdb, executable=executable, core_file=core_file) - long_dump_command = LONG_DUMP_COMMAND.format(gdb=gdb, executable=executable, core_file=core_file) - run_gdb_command(short_dump_command, output_file, output_file + '.err') - output_file_full = output_file + ".full" - output_file_full_err = output_file_full + '.err' - run_gdb_command(long_dump_command, output_file_full, output_file_full_err) - except Exception: - logger.exception("Failed to print trace") - return '', '' - - short_backtrace = '' - full_backtrace = '' - with open(output_file) as o, open(output_file_full) as e: - short_backtrace = o.read() - full_backtrace = e.read() - return short_backtrace, full_backtrace diff --git a/library/python/testing/swag/lib/ya.make b/library/python/testing/swag/lib/ya.make deleted file mode 100644 index c1f486f7fa2..00000000000 --- a/library/python/testing/swag/lib/ya.make +++ /dev/null @@ -1,21 +0,0 @@ -PY23_LIBRARY() - -PEERDIR( - contrib/python/six - library/python/protobuf/runtime - library/python/testing/yatest_common -) - -SRCDIR(library/python/testing/swag) - -PY_SRCS( - NAMESPACE library.python.testing.swag - - daemon.py - gdb.py - pathutil.py - ports.py - proto_traversals.py -) - -END() diff --git a/library/python/testing/swag/pathutil.py b/library/python/testing/swag/pathutil.py deleted file mode 100644 index 3f7910eff3e..00000000000 --- a/library/python/testing/swag/pathutil.py +++ /dev/null @@ -1,26 +0,0 @@ -import os -import tempfile - - -def get_valid_filename(filename, dirname): - current_file, counter = filename, 0 - while os.path.exists(os.path.join(dirname, current_file)): - current_file = "%s_%d" % (filename, counter) - counter += 1 - valid_path = os.path.join(dirname, current_file) - os.mknod(valid_path) - return valid_path - - -def get_valid_tmpdir(name, tmp_dir): - current_dir, counter = name, 0 - while os.path.exists(os.path.join(tmp_dir, current_dir)): - current_dir = "%s_%d" % (name, counter) - counter += 1 - os.mkdir(os.path.join(tmp_dir, current_dir)) - return os.path.join(tmp_dir, current_dir) - - -def get_base_tmpdir(name): - tmppath = tempfile.gettempdir() - return get_valid_tmpdir(name, tmppath) diff --git a/library/python/testing/swag/ports.py b/library/python/testing/swag/ports.py deleted file mode 100644 index 1bf790d06b5..00000000000 --- a/library/python/testing/swag/ports.py +++ /dev/null @@ -1,30 +0,0 @@ -import random -import socket - -ATTEMPTS = 25 -# range 10000-10199 is reserved for Skynet on Sandbox machines -MIN_PORT = 10200 -MAX_PORT = 25000 - - -def is_port_open(host, port): - _socket = socket.socket(socket.AF_INET) - return _socket.connect_ex((host, port)) != 0 - - -def find_free_port(range_start=MIN_PORT, range_end=MAX_PORT, attempts=ATTEMPTS): - """ - Finds free port - - :param range_start: start of range - :param range_end: end of range - :param attempts: number of tries to find free port - - :return: some open port in a given range - """ - ports = [random.randint(range_start, range_end) for _ in range(attempts)] - while ports: - port = ports.pop() - if is_port_open('', port): - return port - raise RuntimeError('Could not find free port in range = ' + str((range_start, range_end))) diff --git a/library/python/testing/swag/proto_traversals.py b/library/python/testing/swag/proto_traversals.py deleted file mode 100644 index 2c521dec918..00000000000 --- a/library/python/testing/swag/proto_traversals.py +++ /dev/null @@ -1,74 +0,0 @@ -import copy -from google.protobuf.descriptor import FieldDescriptor as fdescriptor - -"""Recursive tree traversals for protobuf. Each message - is node, each field is leaf. Function walks through - proto and in each node do smth.""" - - -def search(proto, fname=None, ftype=None): - for desc, obj in proto.ListFields(): - if desc.name == fname and (ftype is None or ftype == desc.type): - return (obj, desc, proto) - if desc.type == fdescriptor.TYPE_MESSAGE: - objs = obj if desc.label == fdescriptor.LABEL_REPEATED else [obj] - for one_obj in objs: - return search(one_obj, fname, ftype) - return None - - -def search_and_process(proto, return_func=lambda params, child_values=None: params, - recalc_params_func=lambda proto, obj, desc, params: params, - params=None): - """Search and process each node. Recalc params on each step. Pass it down - the tree. On each leaf calcs return value from param, and pass it up. Nodes - calc return value with current param and childs return values. - - Args: - * proto -- current node. to run through some proto, put its object here - * return_func -- function that return value. takes current (recalced for current - * node) param and list of return values for current node children. - * for leafs second parametr is None - * recalc_params_func -- function to recalc params in node. takes root proto, - * current object (or objects for repeated fields), current - * proto descriptor and param. return new param value - * params -- initial values for params""" - if proto is None: - return None - - return_values = [] - for desc, obj in proto.ListFields(): - params = copy.deepcopy(params) - if desc.type == fdescriptor.TYPE_MESSAGE: - objs = obj if desc.label == fdescriptor.LABEL_REPEATED else [obj] - params = recalc_params_func(proto, obj, desc, params) - for one_obj in objs: - return_values.append(search_and_process(one_obj, return_func, - recalc_params_func, params)) - else: - return_values.append(return_func(recalc_params_func(proto, obj, desc, params), None)) - return return_func(params, return_values) - - -def search_and_process_descriptors(proto_desc, - return_func=lambda params, child_values=None: params, - recalc_params_func=lambda desc, params: params, - params=None): - """Same as search and process(except we run recalc_params in root_proto too), - but process each node from PROTOBUF DESCRIPTIO, instead of each node from - protobuf message.""" - params = copy.deepcopy(params) - params = recalc_params_func(proto_desc, params) - - if proto_desc is None: - return None - elif hasattr(proto_desc, "type") and proto_desc.type != fdescriptor.TYPE_MESSAGE: - return return_func(params, None) - - return_values = [] - for field_desc in proto_desc.fields: - desc = field_desc if field_desc.message_type is None else field_desc.message_type - return_values.append(search_and_process_descriptors(desc, return_func, - recalc_params_func, params)) - - return return_func(params, return_values) diff --git a/yt/python/yt/__init__.py b/yt/python/yt/__init__.py deleted file mode 100644 index b7cffee5e3c..00000000000 --- a/yt/python/yt/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Python interfaces for YT system. - -YT is reliable and performant distributed storage and computation platform. -There are Python interfaces for low level YT client API: binding to C++ Driver API, wrapper for HTTP-interface and -some pretty "binaries" for interactive usage in command line. -""" diff --git a/yt/python/yt/common.py b/yt/python/yt/common.py deleted file mode 100644 index a646db661ee..00000000000 --- a/yt/python/yt/common.py +++ /dev/null @@ -1,938 +0,0 @@ -try: - from library.python.prctl import prctl -except ImportError: - prctl = None - -# Fix for thread unsafety of datetime module. -# See http://bugs.python.org/issue7980 for more details. -import _strptime # noqa - -# Python3 compatibility -try: - from collections.abc import Mapping -except ImportError: - from collections import Mapping -import datetime -from itertools import chain -from functools import wraps - -import calendar -import copy -import ctypes -import errno -import functools -import inspect -# Intentionally use python-native json module because custom JSONEncoder is required. -import json -import os -import re -import signal -import socket -import sys -import time -import types -import typing -import string -import warnings - -# Standard YT time representation - -YT_DATETIME_FORMAT_STRING = "%Y-%m-%dT%H:%M:%S.%fZ" - -YT_NULL_TRANSACTION_ID = "0-0-0-0" - -_T = typing.TypeVar('_T') - -BYTES_PRINTABLE = set(bytes(string.printable, "ascii")) - - -# Deprecation stuff. -class YtDeprecationWarning(DeprecationWarning): - """Custom warnings category, because built-in category is ignored by default.""" - - -warnings.simplefilter("default", category=YtDeprecationWarning) - -DEFAULT_DEPRECATION_MESSAGE = "{0} is deprecated and will be removed in the next major release, " \ - "use {1} instead" - -ERROR_TEXT_MATCHING_DEPRECATION_MESSAGE = "Matching errors by their messages using string patterns is highly " \ - "discouraged. It is recommended to use contains_code(code) method instead. " \ - "If there is no suitable error code for your needs, ask yt@ for creating one." - - -def declare_deprecated(functional_name, alternative_name, condition=None, message=None): - if condition or condition is None: - message = get_value(message, DEFAULT_DEPRECATION_MESSAGE.format(functional_name, alternative_name)) - warnings.warn(message, YtDeprecationWarning) - - -def deprecated_with_message(message): - def function_decorator(func): - @wraps(func) - def deprecated_function(*args, **kwargs): - warnings.warn(message, YtDeprecationWarning) - return func(*args, **kwargs) - return deprecated_function - return function_decorator - - -def deprecated(alternative): - def function_decorator(func): - warn_message = DEFAULT_DEPRECATION_MESSAGE.format(func.__name__, alternative) - return deprecated_with_message(warn_message)(func) - return function_decorator - - -def get_fqdn(): - fqdn = socket.getfqdn() - if fqdn == "localhost.localdomain": - fqdn = "localhost" - return fqdn - - -class YtError(Exception): - """Base class for all YT errors.""" - def __init__(self, message="", code=1, inner_errors=None, attributes=None): - self.message = message - self.code = code - self.inner_errors = inner_errors if inner_errors is not None else [] - self.attributes = attributes if attributes else {} - if "host" not in self.attributes: - self.attributes["host"] = self._get_fqdn() - if "datetime" not in self.attributes: - self.attributes["datetime"] = datetime_to_string(utcnow()) - - def simplify(self): - """Transforms error (with inner errors) to standard python dict.""" - result = {"message": self.message, "code": self.code} - if self.attributes: - result["attributes"] = self.attributes - if self.inner_errors: - result["inner_errors"] = [] - for error in self.inner_errors: - result["inner_errors"].append( - error.simplify() if isinstance(error, YtError) else - error) - return result - - @classmethod - def from_dict(cls, dict_): - """Restores YtError instance from standard python dict. Reverses simplify().""" - inner_errors = [cls.from_dict(inner) for inner in dict_.get("inner_errors", [])] - return cls(message=dict_["message"], code=dict_["code"], attributes=dict_.get("attributes"), - inner_errors=inner_errors) - - def find_matching_error(self, code=None, predicate=None): - """ - Find a suberror contained in the error (possibly the error itself) which is either: - - having error code equal to `code'; - - or satisfying custom predicate `predicate'. - - Exactly one condition should be specified. - - Returns either first error matching the condition or None if no matching found. - """ - - if sum(argument is not None for argument in (code, predicate)) != 1: - raise ValueError("Exactly one condition should be specified") - - if code is not None: - predicate = lambda error: int(error.code) == code # noqa - - def find_recursive(error): - # error may be Python dict; if so, transform it to YtError. - if not isinstance(error, YtError): - error = YtError(**error) - - if predicate(error): - return error - for inner_error in error.inner_errors: - inner_result = find_recursive(inner_error) - if inner_result: - return inner_result - return None - - return find_recursive(self) - - def contains_code(self, code): - """Check if error or one of its inner errors contains specified error code.""" - return self.find_matching_error(code=code) is not None - - def _contains_text(self, text): - """Inner method, do not call explicitly.""" - return self.find_matching_error(predicate=lambda error: text in error.message) is not None - - @deprecated_with_message(ERROR_TEXT_MATCHING_DEPRECATION_MESSAGE) - def contains_text(self, text): - """ - Check if error or one of its inner errors contains specified substring in message. - - It is not recommended to use this helper; consider using contains_code instead. - If the error you are seeking is not distinguishable by code, please send a message to yt@ - and we will fix that. - """ - - return self._contains_text(text) - - def _matches_regexp(self, pattern): - """Inner method, do not call explicitly.""" - return self.find_matching_error(predicate=lambda error: re.match(pattern, error.message) is not None) is not None - - @deprecated_with_message(ERROR_TEXT_MATCHING_DEPRECATION_MESSAGE) - def matches_regexp(self, pattern): - """ - Check if error message or one of its inner error messages matches given regexp. - - It is not recommended to use this helper; consider using contains_code instead. - If the error you are seeking is not distinguishable by code, please send a message to yt@ - and we will fix that. - """ - - return self._matches_regexp(pattern) - - def __str__(self): - return format_error(self) - - def __repr__(self): - return "%s(%s)" % ( - self.__class__.__name__, - _pretty_format_messages_flat(self)) - - @staticmethod - def _get_fqdn(): - if not hasattr(YtError, "_cached_fqdn"): - YtError._cached_fqdn = get_fqdn() - return YtError._cached_fqdn - - # Error differentiation methods. - def is_retriable_archive_error(self): - """ - Operation progress in Cypress is outdated or some attributes are archive only - while archive request failed - """ - return self.contains_code(1911) - - def is_resolve_error(self): - """Resolution error.""" - return self.contains_code(500) - - def is_already_exists(self): - """Already exists.""" - return self.contains_code(501) - - def is_authentication_error(self): - """Authentication error.""" - return self.contains_code(900) - - def is_access_denied(self): - """Access denied.""" - return self.contains_code(901) - - def is_account_limit_exceeded(self): - """Access denied.""" - return self.contains_code(902) - - def is_concurrent_transaction_lock_conflict(self): - """Deprecated! Transaction lock conflict.""" - return self.contains_code(402) - - def is_cypress_transaction_lock_conflict(self): - """Transaction lock conflict.""" - return self.contains_code(402) - - def is_tablet_transaction_lock_conflict(self): - """Transaction lock conflict.""" - return self.contains_code(1700) - - @deprecated(alternative='use is_request_queue_size_limit_exceeded') - def is_request_rate_limit_exceeded(self): - """Request rate limit exceeded.""" - return self.contains_code(904) - - def is_safe_mode_enabled(self): - """Safe mode enabled.""" - return self.contains_code(906) - - def is_request_queue_size_limit_exceeded(self): - """Request rate limit exceeded.""" - return self.contains_code(108) or self.contains_code(904) - - def is_rpc_unavailable(self): - """Rpc unavailable.""" - return self.contains_code(105) - - def is_proxy_banned(self): - """Proxy is banned.""" - return self.contains_code(2100) - - def is_rpc_response_memory_pressure(self): - """Rpc response memory pressure.""" - return self.contains_code(122) - - def is_master_communication_error(self): - """Master communication error.""" - return self.contains_code(712) - - def is_chunk_unavailable(self): - """Chunk unavailable.""" - return self.contains_code(716) - - def is_request_timed_out(self): - """Request timed out.""" - return self.contains_code(3) - - def is_concurrent_operations_limit_reached(self): - """Too many concurrent operations.""" - return self.contains_code(202) - - def is_master_disconnected(self): - """Master disconnected error.""" - return self.contains_code(218) - - def is_no_such_transaction(self): - """No such transaction.""" - return self.contains_code(11000) - - def is_no_such_job(self): - """No such job.""" - return self.contains_code(203) - - def is_no_such_operation(self): - """No such operation.""" - return self.contains_code(1915) - - def is_shell_exited(self): - """Shell exited.""" - return self.contains_code(1800) or self.contains_code(1801) - - def is_no_such_service(self): - """No such service.""" - return self.contains_code(102) - - def is_transport_error(self): - """Transport error.""" - return self.contains_code(100) - - def is_tablet_in_intermediate_state(self): - """Tablet is in intermediate state.""" - # TODO(ifsmirnov) migrate to error code, YT-10993 - return self.contains_code(1744) or self._matches_regexp("Tablet .* is in state .*") - - def is_hunk_tablet_store_toggle_conflict(self): - """Hunk tablet store toggle conflict.""" - return self.contains_code(1745) - - def is_hunk_store_allocation_failed(self): - """Hunk store allocation failed.""" - return self.contains_code(1746) - - def is_no_such_tablet(self): - """No such tablet.""" - return self.contains_code(1701) - - def is_tablet_not_mounted(self): - """Tablet is not mounted.""" - return self.contains_code(1702) - - def is_all_writes_disabled(self): - """All writes are disabled.""" - return self.contains_code(1703) - - def is_no_such_cell(self): - """No such cell.""" - return self.contains_code(1721) - - def is_all_target_nodes_failed(self): - """Failed to write chunk since all target nodes have failed.""" - return self.contains_code(700) - - def is_no_such_attribute(self, attributes_list=None): - """Operation attribute is not supported.""" - if attributes_list is None: - pred_new = lambda err: err.code == 1920 # noqa - else: - pred_new = lambda err: (err.attributes.get("attribute_name") in attributes_list) and (err.code == 1920) # noqa - pred_old = lambda err: ("Attribute" in err.message) and ("is not allowed" in err.message) # noqa - # COMPAT: remove old version - return self.find_matching_error(predicate=pred_new) or self.find_matching_error(predicate=pred_old) - - def is_row_is_blocked(self): - """Row is blocked.""" - return self.contains_code(1712) - - def is_blocked_row_wait_timeout(self): - """Timed out waiting on blocked row.""" - return self.contains_code(1713) - - def is_chunk_not_preloaded(self): - """Chunk data is not preloaded yet.""" - return self.contains_code(1735) - - def is_no_in_sync_replicas(self): - """No in-sync replicas found.""" - return self.contains_code(1736) - - def is_already_present_in_group(self): - """Member is already present in group.""" - return self.contains_code(908) - - def is_prerequisite_check_failed(self): - """Prerequisite check failed.""" - return self.contains_code(1000) - - def is_prohibited_cross_cell_copy(self): - """Cross-cell "copy"/"move" command is explicitly disabled.""" - return self.contains_code(1002) - - def is_backup_checkpoint_rejected(self): - """Backup checkpoint rejected.""" - return self.contains_code(1733) - - def is_sorting_order_violated(self): - """Sort order violation.""" - return self.contains_code(301) - - def is_command_not_supported(self): - """Command is not supported/Command is not supported by api.""" - return self.contains_text("is not supported") - - -class YtResponseError(YtError): - """Represents an error in YT response.""" - def __init__(self, underlying_error): - super(YtResponseError, self).__init__() - self.message = "Received response with error" - self._underlying_error = underlying_error - self.inner_errors = [self._underlying_error] - - # Common response error properties. - @property - def params(self): - return self.attributes.get("params") - - # HTTP response interface. - @property - def url(self): - """ Returns url for HTTP response error""" - return self.attributes.get("url") - - @property - def headers(self): - """ COMPAT: Returns request headers for HTTP response error""" - return self.attributes.get("request_headers") - - @property - def error(self): - """ COMPAT: Returns underlying error""" - return self._underlying_error - - @property - def request_headers(self): - """ Returns request headers for HTTP response error""" - return self.attributes.get("request_headers") - - @property - def response_headers(self): - """ Returns response headers for HTTP response error""" - return self.attributes.get("response_headers") - - def __reduce__(self): - return (_reconstruct_yt_response_error, (type(self), self.message, self.attributes, self._underlying_error, self.inner_errors)) - - -def _reconstruct_yt_response_error(error_class, message, attributes, underlying_error, inner_errors): - error = error_class(underlying_error) - error.message = message - error.inner_errors = inner_errors - error.attributes = attributes - return error - - -class PrettyPrintableDict(dict): - pass - - -def _pretty_format_escape(value): - def escape(char): - if char in string.printable: - return char - return "\\x{0:02x}".format(ord(char)) - value = value.replace("\n", "\\n").replace("\t", "\\t") - try: - value.encode("utf-8") - return value - except UnicodeDecodeError: - return "".join(map(escape, value)) - - -def _pretty_format_bytes(value): - def escape(byte): - if byte in BYTES_PRINTABLE: - return chr(byte) - return "\\x{0:02x}".format(byte) - - try: - return value.decode("utf-8") - except UnicodeDecodeError: - return "".join(map(escape, value)) - - -def _pretty_format_attribute(name, value, attribute_length_limit): - name = to_native_str(name) - if isinstance(value, PrettyPrintableDict): - class BytesEncoder(json.JSONEncoder): - def default(self, obj): - if isinstance(obj, bytes): - return _pretty_format_bytes(obj) - else: - return super().default(obj) - - value = json.dumps(value, indent=2, cls=BytesEncoder) - value = value.replace("\n", "\n" + " " * (15 + 1 + 4)) - else: - # YsonStringProxy attribute formatting. - if hasattr(value, "_bytes"): - value = value._bytes - else: - if isinstance(value, str): - value = to_native_str(value) - else: - value = str(value) - value = _pretty_format_escape(value) - if attribute_length_limit is not None and len(value) > attribute_length_limit: - value = value[:attribute_length_limit] + "...message truncated..." - return " " * 4 + "%-15s %s" % (name, value) - - -def _pretty_simplify_error(error): - if isinstance(error, YtError): - error = error.simplify() - elif isinstance(error, (Exception, KeyboardInterrupt)): - error = {"code": 1, "message": str(error)} - return error - - -def _pretty_extract_messages(error, depth=0): - """ - YtError -> [(depth: int, message: str), ...], in tree order. - """ - error = _pretty_simplify_error(error) - - if not error.get("attributes", {}).get("transparent", False): - yield (depth, to_native_str(error["message"])) - depth += 1 - - for inner_error in error.get("inner_errors", []): - for subitem in _pretty_extract_messages(inner_error, depth=depth): - yield subitem - - -def _pretty_format_messages_flat(error): - prev_depth = 0 - result = [] - for depth, message in _pretty_extract_messages(error): - if depth > prev_depth: - result.append(" ") - result.append("(" * (depth - prev_depth)) - elif prev_depth > depth: - result.append(")" * (prev_depth - depth)) - elif result: - result.append(", ") - result.append(repr(message)) - prev_depth = depth - - result.append(")" * prev_depth) - return "".join(result) - - -def _pretty_format_messages(error, indent=0, indent_step=4): - result = [] - for depth, message in _pretty_extract_messages(error): - result.append("{indent}{message}".format( - indent=" " * (indent + depth * indent_step), - message=message)) - - return "\n".join(result) - - -def _pretty_format_full_errors(error, attribute_length_limit): - error = _pretty_simplify_error(error) - - lines = [] - if "message" in error: - lines.append(to_native_str(error["message"])) - - if "code" in error and int(error["code"]) != 1: - lines.append(_pretty_format_attribute( - "code", error["code"], attribute_length_limit=attribute_length_limit)) - - attributes = error.get("attributes", {}) - - origin_keys = ["host", "datetime"] - origin_cpp_keys = ["pid", "tid", "fid"] - if all(key in attributes for key in origin_keys): - date = attributes["datetime"] - if isinstance(date, datetime.datetime): - date = date.strftime("%y-%m-%dT%H:%M:%S.%fZ") - value = "{0} on {1}".format(attributes["host"], date) - if all(key in attributes for key in origin_cpp_keys): - value += " (pid %(pid)d, tid %(tid)x, fid %(fid)x)" % attributes - lines.append(_pretty_format_attribute( - "origin", value, attribute_length_limit=attribute_length_limit)) - - location_keys = ["file", "line"] - if all(key in attributes for key in location_keys): - lines.append(_pretty_format_attribute( - "location", - "%(file)s:%(line)d" % attributes, - attribute_length_limit=attribute_length_limit)) - - for key, value in attributes.items(): - if key in origin_keys or key in location_keys or key in origin_cpp_keys: - continue - lines.append(_pretty_format_attribute( - key, value, attribute_length_limit=attribute_length_limit)) - - result = (" " * 4 + "\n").join(lines) - if "inner_errors" in error: - for inner_error in error["inner_errors"]: - result += "\n" + _pretty_format_full_errors( - inner_error, attribute_length_limit=attribute_length_limit) - - return result - - -def _pretty_format(error, attribute_length_limit=None): - return "{}\n\n***** Details:\n{}\n".format( - _pretty_format_messages(error), - _pretty_format_full_errors(error, attribute_length_limit=attribute_length_limit)) - - -def _pretty_format_fake(error, attribute_length_limit=None): - return _pretty_format(error, attribute_length_limit) - - -def _pretty_format_for_logging(error, attribute_length_limit=None): - return _pretty_format_full_errors(error, attribute_length_limit=attribute_length_limit).replace("\n", "\\n") - - -def format_error(error, attribute_length_limit=300): - return _pretty_format(error, attribute_length_limit) - - -def join_exceptions(*args): - result = [] - for exception in args: - if isinstance(exception, tuple): - result += exception - else: - result.append(exception) - return tuple(result) - - -def which(name, flags=os.X_OK, custom_paths=None): - """Return list of files in system paths with given name.""" - # TODO: check behavior when dealing with symlinks - result = [] - paths = os.environ.get("PATH", "").split(os.pathsep) - if custom_paths is not None: - paths = custom_paths + paths - for dir in paths: - path = os.path.join(dir, name) - if os.access(path, flags): - result.append(path) - return result - - -def unlist(list): - try: - return list[0] if len(list) == 1 else list - except TypeError: # cannot calculate len - return list - - -def require(condition, exception_func): - if not condition: - raise exception_func() - - -def update_inplace(object: _T, patch) -> _T: - """Apply patch to object inplace""" - if isinstance(patch, Mapping) and isinstance(object, Mapping): - for key, value in patch.items(): - if key in object: - object[key] = update_inplace(object[key], value) - else: - object[key] = value - elif isinstance(patch, list) and isinstance(object, list): - for index, value in enumerate(patch): - if index < len(object): - object[index] = update_inplace(object[index], value) - else: - object.append(value) - else: - object = patch - return object - - -def update(object: _T, patch) -> _T: - """Apply patch to object without modifying original object or patch""" - if patch is None: - return copy.deepcopy(object) - elif object is None: - return copy.deepcopy(patch) - else: - return update_inplace(copy.deepcopy(object), patch) - - -def flatten(obj, list_types=(list, tuple, set, frozenset, types.GeneratorType)): - """Create flat list from all elements.""" - if isinstance(obj, list_types): - return list(chain(*map(flatten, obj))) - return [obj] - - -def update_from_env(variables): - """Update variables dict from environment (cuts name prefix "YT_").""" - for key, value in os.environ.items(): - prefix = "YT_" - if not key.startswith(prefix): - continue - - key = key[len(prefix):] - if key not in variables: - continue - - var_type = type(variables[key]) - # Using int we treat "0" as false, "1" as "true" - if var_type == bool: - try: - value = int(value) - except: # noqa - pass - # None type is treated as str - if isinstance(None, var_type): - var_type = str - - variables[key] = var_type(value) - - -def get_value(value, default): - if value is None: - return default - else: - return value - - -def filter_dict(predicate, dictionary): - return dict([(k, v) for (k, v) in dictionary.items() if predicate(k, v)]) - - -def set_pdeathsig(signum=None): - if sys.platform.startswith("linux"): - if signum is None: - signum = signal.SIGTERM - if prctl: - prctl.set_pdeathsig(signum) - else: - ctypes.cdll.LoadLibrary("libc.so.6") - libc = ctypes.CDLL("libc.so.6") - PR_SET_PDEATHSIG = 1 - libc.prctl(PR_SET_PDEATHSIG, signum) - - -def remove_file(path, force=False): - try: - os.remove(path) - except OSError: - if not force: - raise - - -def makedirp(path): - try: - os.makedirs(path) - except OSError as err: - if err.errno != errno.EEXIST: - raise - - -def touch(path): - if not os.path.exists(path): - makedirp(os.path.dirname(path)) - with open(path, "w"): - pass - - -def date_string_to_datetime(date): - return datetime.datetime.strptime(date, YT_DATETIME_FORMAT_STRING) - - -def date_string_to_timestamp(date): - return calendar.timegm(date_string_to_datetime(date).timetuple()) - - -def date_string_to_timestamp_mcs(time_str): - dt = date_string_to_datetime(time_str) - return int(calendar.timegm(dt.timetuple()) * (10 ** 6) + dt.microsecond) - - -def datetime_to_string(date, is_local=False): - if is_local: - date = datetime.datetime.utcfromtimestamp(time.mktime(date.timetuple())) - return date.strftime(YT_DATETIME_FORMAT_STRING) - - -def utcnow(): - if sys.version_info >= (3, 12, ): - return datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) - else: - return datetime.datetime.utcnow() - - -def make_non_blocking(fd): - # Use local import to support Windows. - import fcntl - flags = fcntl.fcntl(fd, fcntl.F_GETFL) - fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) - - -def to_native_str(string, encoding="utf-8", errors="strict"): - if isinstance(string, bytes): - return string.decode(encoding, errors=errors) - return string - - -def copy_docstring_from(documented_function): - """Decorator that copies docstring from one function to another. - - :param documented_function: function that provides docstring. - - Usage:: - - def foo(...): - "USEFUL DOCUMENTATION" - ... - - @copy_docstring_from(foo) - def bar(...) - # docstring will be copied from `foo' function - ... - """ - return functools.wraps(documented_function, assigned=("__doc__",), updated=()) - - -def is_process_alive(pid): - try: - os.kill(pid, 0) - except OSError as err: - if err.errno == errno.ESRCH: - return False - elif err.errno == errno.EPERM: - return True - else: - # According to "man 2 kill" possible error values are - # (EINVAL, EPERM, ESRCH) - raise - return True - - -def uuid_to_parts(guid): - id_parts = guid.split("-") - id_hi = int(id_parts[2], 16) << 32 | int(id_parts[3], 16) - id_lo = int(id_parts[0], 16) << 32 | int(id_parts[1], 16) - return id_hi, id_lo - - -def parts_to_uuid(id_hi, id_lo): - guid = id_lo << 64 | id_hi - mask = 0xFFFFFFFF - - parts = [] - for i in range(4): - parts.append((guid & mask) >> (i * 32)) - mask <<= 32 - - return "-".join(reversed(["{:x}".format(part) for part in parts])) - - -# TODO(asaitgalin): Remove copy-paste from YP. -def underscore_case_to_camel_case(str): - result = [] - first = True - upper = True - for c in str: - if c == "_": - upper = True - else: - if upper: - if c not in string.ascii_letters and not first: - result.append("_") - c = c.upper() - result.append(c) - upper = False - first = False - return "".join(result) - - -class WaitFailed(Exception): - pass - - -def wait(predicate, error_message=None, iter=None, sleep_backoff=None, timeout=None, ignore_exceptions=False): - # 30 seconds by default - if sleep_backoff is None: - sleep_backoff = 0.3 - - last_exception = None - if ignore_exceptions: - def check_predicate(): - try: - return predicate(), None, None - # Do not catch BaseException because pytest exceptions are inherited from it - # pytest.fail raises exception inherited from BaseException. - except Exception as ex: - ex_traceback = sys.exc_info()[2] - return False, ex, ex_traceback - else: - def check_predicate(): - return predicate(), None, None - - result = False - last_exception = None - last_exception_traceback = None - - if timeout is None: - if iter is None: - iter = 100 - index = 0 - while index < iter: - result, last_exception, last_exception_traceback = check_predicate() - if result: - return - index += 1 - time.sleep(sleep_backoff) - else: - start_time = datetime.datetime.now() - while datetime.datetime.now() - start_time < datetime.timedelta(seconds=timeout): - result, last_exception, last_exception_traceback = check_predicate() - if result: - return - time.sleep(sleep_backoff) - - if inspect.isfunction(error_message): - error_message = error_message() - if error_message is None: - error_message = "Wait failed" - - error_message += f" (timeout = {timeout if timeout is not None else iter * sleep_backoff}" - if last_exception is not None: - error_message += f", exception = {last_exception}" - error_message += ")" - - ex = WaitFailed(error_message) - if last_exception_traceback is None: - raise ex - else: - raise ex.with_traceback(last_exception_traceback) diff --git a/yt/python/yt/json_wrapper.py b/yt/python/yt/json_wrapper.py deleted file mode 100644 index a1632a1263e..00000000000 --- a/yt/python/yt/json_wrapper.py +++ /dev/null @@ -1,19 +0,0 @@ -try: - from simplejson import load, dump, loads, dumps, JSONDecodeError # noqa -except ImportError: - # This version of simplejson has no compiled speedup module. - from yt.packages.simplejson import load, dump, loads, dumps, JSONDecodeError # noqa - - -def loads_as_bytes(*args, **kwargs): - def encode(value): - if isinstance(value, dict): - return dict([(encode(k), encode(v)) for k, v in value.items()]) - elif isinstance(value, list): - return [encode(item) for item in value] - elif isinstance(value, str): - return value.encode("utf-8") - else: - return value - - return encode(loads(*args, **kwargs)) diff --git a/yt/python/yt/logger.py b/yt/python/yt/logger.py deleted file mode 100644 index 58a8bb3af66..00000000000 --- a/yt/python/yt/logger.py +++ /dev/null @@ -1,146 +0,0 @@ -from . import logger_config - -try: - import yatest.common as yatest_common -except ImportError: - yatest_common = None - -import functools -import hashlib -import logging -import os -import re -import time - -LOG_ONCE_BUFF = {} -MAX_BUFF_LEN = 1000 -BUFF_CLEANING_LEN = 10 - - -def set_log_level_from_config(logger): - if not logger_config.LOG_LEVEL: - logger.setLevel(level=logging.__dict__["INFO"]) - else: - if logger_config.LOG_LEVEL.upper() == "NOTSET": - raise Exception("LOG_LEVEL couldn't be defined as NOTSET") - # Intentionally override trace with debug for compatibility with C++ logging library. - if logger_config.LOG_LEVEL.upper() == "TRACE": - logger_config.LOG_LEVEL = "DEBUG" - logger.setLevel(level=logging.__dict__[logger_config.LOG_LEVEL.upper()]) - - -class SimpleColorizedStreamHandler(logging.StreamHandler): - C_LCYAN = "\033[96m" - C_LBLUE = "\033[94m" - C_LGREEN = "\033[92m" - C_LYELLOW = "\033[93m" - C_LGRAY = "\033[37m" - C_BOLD = "\033[1m" - C_END = "\033[0m" - - KW = C_LBLUE - URL = C_LCYAN + C_BOLD - PARAM = C_LGRAY - YSON_PARAM = C_LYELLOW - - RE_KW = functools.partial(lambda p, r, m: p.sub(r, m), re.compile(r"(Perform HTTP \S+ request|Response received)"), r"{}\1{}".format(KW, C_END)) - RE_HTTP = functools.partial(lambda p, r, m: p.sub(r, m), re.compile(r"(https?://\S+)"), r"{}\1{}".format(URL, C_END)) - RE_JSON = functools.partial(lambda p, r, m: p.sub(r, m), re.compile(r"(['\"][\w-]+['\"]): ?"), r"{}\1{}: ".format(PARAM, C_END)) - RE_YSON = functools.partial(lambda p, r, m: p.sub(r, m), re.compile(r"\"([^\";]+?)\"="), "\"{}\\1{}\"=".format(YSON_PARAM, C_END)) - - # to disable color output - DISABLE_COLOR = os.environ.get("YT_LOG_LEVEL") == "Debug" - - terminator = '\n' # py2 compat - - def _colorize(self, msg): - msg = SimpleColorizedStreamHandler.RE_KW(msg) - msg = SimpleColorizedStreamHandler.RE_HTTP(msg) - msg = SimpleColorizedStreamHandler.RE_JSON(msg) - msg = SimpleColorizedStreamHandler.RE_YSON(msg) - return msg - - def emit(self, record): - try: - msg = self.format(record) - stream = self.stream - if stream.isatty() and record.levelno == logging.DEBUG and not self.DISABLE_COLOR: - msg = self._colorize(msg) - stream.write(msg + self.terminator) - self.flush() - except Exception: - self.handleError(record) - - -formatter = None - - -def set_formatter(new_formatter): - global formatter - formatter = new_formatter - for handler in LOGGER.handlers: - handler.setFormatter(new_formatter) - - -logging.getLogger("yt.packages.urllib3").setLevel(logging.WARNING) - -LOGGER = logging.getLogger("Yt") - -LOGGER.propagate = False - -set_log_level_from_config(LOGGER) - -if logger_config.LOG_PATH is None: - LOGGER.addHandler(SimpleColorizedStreamHandler()) -else: - LOGGER.addHandler(logging.FileHandler(logger_config.LOG_PATH)) - -BASIC_FORMATTER = logging.Formatter(logger_config.LOG_PATTERN) - -set_formatter(BASIC_FORMATTER) - - -def debug(msg, *args, **kwargs): - LOGGER.debug(msg, *args, **kwargs) - - -def info(msg, *args, **kwargs): - LOGGER.info(msg, *args, **kwargs) - - -def warning(msg, *args, **kwargs): - LOGGER.warning(msg, *args, **kwargs) - - -def error(msg, *args, **kwargs): - LOGGER.error(msg, *args, **kwargs) - - -def exception(msg, *args, **kwargs): - LOGGER.exception(msg, *args, **kwargs) - - -if hasattr(functools, 'lru_cache') and not os.environ.get("YT_LOG_NO_TIP"): - @functools.lru_cache(maxsize=128) - def tip(msg): - LOGGER.debug("[TIP] " + msg) -else: - # py2 - def tip(msg): - pass - - -def log(level, msg, *args, **kwargs): - LOGGER.log(level, msg, *args, **kwargs) - - -def log_once(level, msg, *args, **kwargs): - msg_hash = hashlib.sha256(msg.encode('utf-8')).hexdigest() - if msg_hash not in LOG_ONCE_BUFF: - LOGGER.log(level, msg, *args, **kwargs) - LOG_ONCE_BUFF[msg_hash] = time.time() - - if len(LOG_ONCE_BUFF) >= MAX_BUFF_LEN: - cleaning_items = sorted(LOG_ONCE_BUFF.items(), key=lambda i: i[1])[:BUFF_CLEANING_LEN] - for msg, msg_time in cleaning_items: - del LOG_ONCE_BUFF[msg] diff --git a/yt/python/yt/logger_config.py b/yt/python/yt/logger_config.py deleted file mode 100644 index 45d1bdd1a7c..00000000000 --- a/yt/python/yt/logger_config.py +++ /dev/null @@ -1,7 +0,0 @@ -from .common import update_from_env - -LOG_LEVEL = None -LOG_PATTERN = "%(asctime)-15s\t%(levelname)s\t%(message)s" -LOG_PATH = None - -update_from_env(globals()) diff --git a/yt/python/yt/subprocess_wrapper.py b/yt/python/yt/subprocess_wrapper.py deleted file mode 100644 index b766cb11fe5..00000000000 --- a/yt/python/yt/subprocess_wrapper.py +++ /dev/null @@ -1,19 +0,0 @@ -from yt.common import to_native_str - -try: - import subprocess32 as subprocess -except ImportError: - import subprocess - -Popen = subprocess.Popen -PIPE = subprocess.PIPE -STDOUT = subprocess.STDOUT -CalledProcessError = subprocess.CalledProcessError - - -def check_call(*args, **kwargs): - return subprocess.check_call(*args, **kwargs) - - -def check_output(*args, **kwargs): - return to_native_str(subprocess.check_output(*args, **kwargs)) diff --git a/yt/python/yt/ya.make b/yt/python/yt/ya.make deleted file mode 100644 index 9aea3b8e7d6..00000000000 --- a/yt/python/yt/ya.make +++ /dev/null @@ -1,31 +0,0 @@ -PY23_LIBRARY() - -INCLUDE(${ARCADIA_ROOT}/yt/opensource.inc) - -IF (PYTHON2) - PEERDIR(yt/python_py2/yt) -ELSE() - PEERDIR( - contrib/python/simplejson - yt/python/yt/type_info - ) - - IF(LINUX) - PEERDIR( - library/python/prctl - ) - ENDIF() - - PY_SRCS( - NAMESPACE yt - - __init__.py - common.py - json_wrapper.py - logger.py - logger_config.py - subprocess_wrapper.py - ) -ENDIF() - -END() diff --git a/yt/python/yt/yson/__init__.py b/yt/python/yt/yson/__init__.py deleted file mode 100644 index 7ca6f7960be..00000000000 --- a/yt/python/yt/yson/__init__.py +++ /dev/null @@ -1,101 +0,0 @@ -""" -YSON library. - -Package supports `YT YSON format <https://ytsaurus.tech/docs/en/user-guide/storage/yson>`_. - -Package provides special classes for all yson types, see :mod:`yson_types <yt.yson.yson_types>` module. -Also it provides methods for serialization and deserialization yson data: -:func:`load <yt.yson.parser.load>`, :func:`loads <yt.yson.parser.loads>`, -:func:`dump <yt.yson.writer.dump>`, :func:`dumps <yt.yson.writer.dumps>`. -And finally it provides method :func:`to_yson_type <yt.yson.convert.to_yson_type>` for conversion -python objects to special yson types. - -In special variable `TYPE` you can find implementation type of the library. -In equals "BINARY" if c++ bindings found and "PYTHON" otherwise. - -Examples: - ->>> import yt.yson as yson ->>> yson.loads("{a=10}") -{'a': 10} - ->>> yson.dumps(True) -'"true"' - ->>> number = yson.YsonInt64(10) ->>> number.attributes["my_attr"] = "hello" ->>> yson.dumps(number) -'<"attr"="hello">10' - ->>> boolean = to_yson_type(False, attributes={"my_attr": "my_value"}) - -""" - -from __future__ import print_function - -from . import writer # noqa -from . import parser # noqa -from . import yson_types # noqa - -TYPE = None -HAS_PARQUET = False -HAS_ORC = False - -try: - from yt_yson_bindings import load, loads, dump, dumps # noqa - TYPE = "BINARY" -except ImportError as error: - # XXX(asaitgalin): Sometimes module can't be imported because - # it depends on missing dynamic libraries (e.g. libatomic). In this case - # diagnostic is printed to stderr. - message = str(error) - if "No module named" not in message: - import sys as _sys - print("Warning! Failed to import YSON bindings: " + message, file=_sys.stderr) - -try: - from yt_yson_bindings import upload_parquet, dump_parquet, async_dump_parquet # noqa - HAS_PARQUET = True -except Exception: - pass - -try: - from yt_yson_bindings import dump_orc, upload_orc, async_dump_orc # noqa - HAS_ORC = True -except Exception: - pass - -if TYPE is None: - from .parser import load, loads # noqa - from .writer import dump, dumps # noqa - TYPE = "PYTHON" - -from .yson_types import ( # noqa - YsonString, YsonUnicode, YsonInt64, YsonUint64, YsonDouble, - YsonBoolean, YsonList, YsonMap, YsonEntity, YsonType, YsonStringProxy, - is_unicode, get_bytes, make_byte_key) - -from .convert import to_yson_type, yson_to_json, json_to_yson # noqa -from .common import YsonError # noqa - - -def _loads_from_native_str(string, encoding="utf-8", **kwargs): - import sys - - if sys.version_info[0] < 3: - return loads(string, **kwargs) - - if isinstance(string, str): - string = string.encode(encoding) - - return loads(string, encoding=encoding, **kwargs) - - -def _dumps_to_native_str(obj, encoding="utf-8", **kwargs): - import sys - - if sys.version_info[0] < 3: - return dumps(obj, **kwargs) - - s = dumps(obj, encoding=encoding, **kwargs) - return s.decode(encoding) diff --git a/yt/python/yt/yson/common.py b/yt/python/yt/yson/common.py deleted file mode 100644 index 5cddcd8203e..00000000000 --- a/yt/python/yt/yson/common.py +++ /dev/null @@ -1,64 +0,0 @@ -from yt.common import YtError - - -class YsonError(YtError): - pass - - -def raise_yson_error(message, position_info): - line_index, position, offset = position_info - raise YsonError(message, attributes={"line": line_index, "position": position, "offset": offset}) - - -class StreamWrap(object): - def __init__(self, stream, header, footer): - self.stream = stream - self.header = header - self.footer = footer - - self.pos = 0 - self.state = 0 - - def read(self, n): - if n == 0: - return self.stream.read(0) - - assert n == 1 - - if self.state == 0: - if self.pos == len(self.header): - self.state += 1 - else: - res = bytes([self.header[self.pos]]) - self.pos += 1 - return res - - if self.state == 1: - sym = self.stream.read(1) - if sym: - return sym - else: - self.state += 1 - self.pos = 0 - - if self.state == 2: - if self.pos == len(self.footer): - self.state += 1 - else: - res = bytes([self.footer[self.pos]]) - self.pos += 1 - return res - - if self.state == 3: - return b"" - - -_ENCODING_SENTINEL = object() - -# Binary literals markers -STRING_MARKER = bytes([1]) -INT64_MARKER = bytes([2]) -DOUBLE_MARKER = bytes([3]) -FALSE_MARKER = bytes([4]) -TRUE_MARKER = bytes([5]) -UINT64_MARKER = bytes([6]) diff --git a/yt/python/yt/yson/convert.py b/yt/python/yt/yson/convert.py deleted file mode 100644 index 90923818067..00000000000 --- a/yt/python/yt/yson/convert.py +++ /dev/null @@ -1,209 +0,0 @@ -from .yson_types import ( - YsonType, YsonString, YsonUnicode, YsonBoolean, YsonInt64, YsonUint64, YsonDouble, - YsonList, YsonMap, YsonEntity, YsonStringProxy, get_bytes) -from .common import YsonError - -import copy - - -def to_yson_type(value, attributes=None, always_create_attributes=True, encoding="utf-8"): - """Wraps value with YSON type.""" - if not always_create_attributes and attributes is None: - return value - - if isinstance(value, YsonType): - if attributes is not None: - value = copy.deepcopy(value) - value.attributes = attributes - return value - - if isinstance(value, str): - result = YsonUnicode(value) - elif isinstance(value, bytes): - result = YsonString(value) - elif value is False or value is True: - result = YsonBoolean(value) - elif isinstance(value, int): - if value < -2 ** 63 or value >= 2 ** 64: - raise TypeError("Integer {0} cannot be represented in YSON " - "since it is out of range [-2^63, 2^64 - 1])".format(value)) - greater_than_max_int64 = value >= 2 ** 63 - if greater_than_max_int64 or isinstance(value, YsonUint64): - result = YsonUint64(value) - else: - result = YsonInt64(value) - elif isinstance(value, float): - result = YsonDouble(value) - elif isinstance(value, list): - result = YsonList(value) - elif isinstance(value, dict): - result = YsonMap(value) - else: - result = YsonEntity() - - if attributes is not None: - result.attributes = attributes - else: - result.attributes = {} - - return result - - -# TODO(ignat): Should we make auto-detection for use_byte_strings? -def json_to_yson(json_tree, use_byte_strings=None): - """Converts json representation to YSON representation.""" - def to_literal(string): - if use_byte_strings: - return string.encode("ascii") - else: - return string - - def decode_key(string): - # In yt wrapper we expect here correct keys, but other usages in arcadia could not give this guarantee. - # TODO(ignat): fix this usages. - if use_byte_strings: - if not isinstance(string, bytes): - string = string.encode("ascii") - else: - if not isinstance(string, str): - string = string.decode("ascii") - - if string.startswith(to_literal("$")): - if not string.startswith(to_literal("$$")): - raise YsonError("Keys should not start with single dollar sign") - string = string[1:] - return string - - if use_byte_strings is None: - use_byte_strings = False - - has_attrs = isinstance(json_tree, dict) and to_literal("$value") in json_tree - value = json_tree[to_literal("$value")] if has_attrs else json_tree - if isinstance(value, str): - result = YsonUnicode(value) - elif isinstance(value, bytes): - result = YsonString(value) - elif value is False or value is True: - result = YsonBoolean(value) - elif isinstance(value, int): - greater_than_max_int64 = value >= 2 ** 63 - if greater_than_max_int64: - result = YsonUint64(value) - else: - result = YsonInt64(value) - elif isinstance(value, float): - result = YsonDouble(value) - elif isinstance(value, list): - result = YsonList(map(lambda item: json_to_yson(item, use_byte_strings=use_byte_strings), value)) - elif isinstance(value, dict): - result = YsonMap((decode_key(k), json_to_yson(v, use_byte_strings=use_byte_strings)) for k, v in YsonMap(value).items()) - elif value is None: - result = YsonEntity() - else: - raise YsonError("Unknown type:", type(value)) - - if has_attrs and json_tree.get(to_literal("$attributes"), {}): - result.attributes = json_to_yson(json_tree[to_literal("$attributes")], use_byte_strings=use_byte_strings) - return result - - -def _yson_to_json(yson_tree, print_attributes=True, attributes_printed=False, annotate_with_types=False, use_byte_strings=False): - should_annotate_with_types = ( - annotate_with_types and - not isinstance(yson_tree, list) and - not isinstance(yson_tree, dict) and - not isinstance(yson_tree, YsonEntity) and - yson_tree is not None - ) - - def encode_key(key): - if isinstance(key, bytes): - key = key.decode("ascii") - if key and key[0] == "$": - return "$" + key - return key - - def process_dict(d): - return dict( - ( - encode_key(k), - _yson_to_json(v, print_attributes=print_attributes, annotate_with_types=annotate_with_types, use_byte_strings=use_byte_strings), - ) for k, v in d.items() - ) - - def get_type_name(): - if isinstance(yson_tree, YsonType): - yson_type_str = yson_tree.get_yson_type_str() - - if yson_type_str is not None: - return yson_type_str - - if isinstance(yson_tree, bool): - return "bool" - elif isinstance(yson_tree, int): - return "int64" - elif isinstance(yson_tree, float): - return "double" - elif isinstance(yson_tree, str): - # TODO: "utf8" - return "string" - elif isinstance(yson_tree, bytes): - return "string" - else: - raise RuntimeError("Failed to perform yson to json conversion of {!r}, unknown type {!r} to annotate with types".format( - yson_tree, - type(yson_tree) - )) - - def do_annotate_with_types(value): - return {"$type": get_type_name(), "$value": value} if should_annotate_with_types else value - - if hasattr(yson_tree, "attributes") and yson_tree.attributes and print_attributes and not attributes_printed: - # If value is primitive do not pass annotate with types. - value_annotate_with_types = False if should_annotate_with_types else annotate_with_types - - result = { - "$attributes": process_dict(yson_tree.attributes), - "$value": _yson_to_json(yson_tree, print_attributes=print_attributes, attributes_printed=True, annotate_with_types=value_annotate_with_types, use_byte_strings=use_byte_strings), - } - - if should_annotate_with_types: - result["$type"] = get_type_name() - - return result - - if isinstance(yson_tree, list): - return [_yson_to_json(element, print_attributes=print_attributes, annotate_with_types=annotate_with_types, use_byte_strings=use_byte_strings) for element in yson_tree] - elif isinstance(yson_tree, dict): - return process_dict(yson_tree) - elif isinstance(yson_tree, YsonEntity): - return None - elif isinstance(yson_tree, YsonStringProxy): - tree_value = get_bytes(yson_tree) - if not use_byte_strings: - tree_value = tree_value.decode("utf-8") - return do_annotate_with_types(tree_value) - elif isinstance(yson_tree, bytes) or isinstance(yson_tree, YsonString): - if use_byte_strings: - tree_value = get_bytes(yson_tree) - else: - tree_value = yson_tree.decode("utf-8") - return do_annotate_with_types(tree_value) - elif isinstance(yson_tree, bool) or isinstance(yson_tree, YsonBoolean): - return do_annotate_with_types(True if yson_tree else False) - else: - bases = type(yson_tree).__bases__ - while len(bases) == 1 and YsonType not in bases: - bases = bases[0].__bases__ - - if YsonType in bases: - other_types = list(set(bases) - set([YsonType])) - if not other_types: - raise RuntimeError("Failed to perform yson to json conversion of {!r}".format(yson_tree)) - other = other_types[0] - return do_annotate_with_types(other(yson_tree)) - return do_annotate_with_types(yson_tree) - - -def yson_to_json(yson_tree, print_attributes=True, annotate_with_types=False, use_byte_strings=False): - return _yson_to_json(yson_tree, print_attributes=print_attributes, annotate_with_types=annotate_with_types, use_byte_strings=use_byte_strings) diff --git a/yt/python/yt/yson/lexer.py b/yt/python/yt/yson/lexer.py deleted file mode 100644 index ce3adf68622..00000000000 --- a/yt/python/yt/yson/lexer.py +++ /dev/null @@ -1,391 +0,0 @@ -from . import yson_types -from .yson_token import ( - YsonToken, - TOKEN_STRING, - TOKEN_INT64, - TOKEN_UINT64, - TOKEN_DOUBLE, - TOKEN_BOOLEAN, - TOKEN_HASH, - TOKEN_LEFT_PARENTHESIS, - TOKEN_RIGHT_PARENTHESIS, - TOKEN_COMMA, - TOKEN_COLON, - TOKEN_SEMICOLON, - TOKEN_LEFT_ANGLE, - TOKEN_EQUALS, - TOKEN_RIGHT_ANGLE, - TOKEN_LEFT_BRACKET, - TOKEN_RIGHT_BRACKET, - TOKEN_LEFT_BRACE, - TOKEN_RIGHT_BRACE) - -from .common import ( - raise_yson_error, _ENCODING_SENTINEL, - STRING_MARKER, INT64_MARKER, DOUBLE_MARKER, - FALSE_MARKER, TRUE_MARKER, UINT64_MARKER) - -import struct - -_SEEMS_INT64 = b'\x00' -_SEEMS_UINT64 = b'\x01' -_SEEMS_DOUBLE = b'\x02' - -PERCENT_LITERALS = [b"true", b"false", b"nan", b"inf", b"-inf", b"+inf"] -PERCENT_LITERAL_LENGTH = dict((s[0:1], len(s)) for s in PERCENT_LITERALS) -assert len(PERCENT_LITERALS) == len(PERCENT_LITERAL_LENGTH) - - -def _get_numeric_type(string): - for code in string: - ch = bytes([code]) - if ch == b"E" or ch == b"e" or ch == b".": - return _SEEMS_DOUBLE - elif ch == b"u": - return _SEEMS_UINT64 - return _SEEMS_INT64 - - -def _zig_zag_decode(value): - return (value >> 1) ^ -(value & 1) - - -class YsonLexer(object): - def __init__(self, stream, encoding=None, output_buffer=None): - assert (encoding is _ENCODING_SENTINEL) != (output_buffer is None), \ - "Exactly one of encoding and output_buffer parameters must be specified" - - self._line_index = 1 - self._position = 1 - self._offset = 0 - self._stream = stream - self._lookahead = None - self._encoding = encoding - self._output_buffer = output_buffer - - def _get_start_state(self, ch): - tokens = { - b"#": TOKEN_HASH, - b"(": TOKEN_LEFT_PARENTHESIS, - b")": TOKEN_RIGHT_PARENTHESIS, - b",": TOKEN_COMMA, - b":": TOKEN_COLON, - b";": TOKEN_SEMICOLON, - b"<": TOKEN_LEFT_ANGLE, - b"=": TOKEN_EQUALS, - b">": TOKEN_RIGHT_ANGLE, - b"[": TOKEN_LEFT_BRACKET, - b"]": TOKEN_RIGHT_BRACKET, - b"{": TOKEN_LEFT_BRACE, - b"}": TOKEN_RIGHT_BRACE, - } - return tokens.get(ch) - - def get_next_token(self): - self._skip_whitespaces() - ch = self._peek_char() - if not ch: - return YsonToken() - - if ch == STRING_MARKER: - return YsonToken(value=self._parse_string(), type=TOKEN_STRING) - - elif ch == b"_" or ch == b'"' or ch.isalpha(): - return YsonToken(value=self._parse_string(), type=TOKEN_STRING) - elif ch == INT64_MARKER: - return YsonToken(value=self._parse_binary_int64(), type=TOKEN_INT64) - - elif ch == UINT64_MARKER: - return YsonToken(value=self._parse_binary_uint64(), type=TOKEN_UINT64) - - elif ch == DOUBLE_MARKER: - return YsonToken(value=self._parse_binary_double(), type=TOKEN_DOUBLE) - - elif ch == FALSE_MARKER: - self._expect_char(ch) - return YsonToken(value=self._maybe_value(False), type=TOKEN_BOOLEAN) - - elif ch == TRUE_MARKER: - self._expect_char(ch) - return YsonToken(value=self._maybe_value(True), type=TOKEN_BOOLEAN) - - elif ch == b"%": - value, token_type = self._parse_percent_literal() - return YsonToken(value=self._maybe_value(value), type=token_type) - - elif ch == b"#": - return YsonToken(value=self._parse_entity(), type=TOKEN_HASH) - - elif ch == b"+" or ch == b"-" or ch.isdigit(): - value, token_type = self._parse_numeric() - return YsonToken(value=self._maybe_value(value), type=token_type) - - state = self._get_start_state(ch) - self._read_char() - return YsonToken(value=self._maybe_value(ch), type=state) - - def get_position_info(self): - return self._line_index, self._position, self._offset - - def _maybe_value(self, value): - if self._output_buffer is None: - return value - return None - - def _read_char(self, binary_input=False): - if self._lookahead is None: - result = self._stream.read(1) - else: - result = self._lookahead - self._lookahead = None - - self._offset += 1 - if not binary_input and result == b"\n": - self._line_index += 1 - self._position = 1 - else: - self._position += 1 - - if self._output_buffer is not None: - self._output_buffer.append(ord(result)) - - return result - - def _peek_char(self): - if self._lookahead is not None: - return self._lookahead - self._lookahead = self._stream.read(1) - return self._lookahead - - def _read_binary_chars(self, char_count): - if self._output_buffer is not None: - string = self._stream.read(char_count) - self._position += len(string) - if len(string) != char_count: - raise_yson_error( - "Premature end-of-stream while reading byte {0} out of {1}".format(len(string) + 1, char_count), - self.get_position_info()) - self._output_buffer += string - return string - - result = [] - for i in range(char_count): - ch = self._read_char(True) - if not ch: - raise_yson_error( - "Premature end-of-stream while reading byte {0} out of {1}".format(i + 1, char_count), - self.get_position_info()) - result.append(ch) - return b"".join(result) - - def _expect_char(self, expected_ch): - read_ch = self._read_char() - if not read_ch: - raise_yson_error( - 'Premature end-of-stream expecting "{0}" in Yson'.format(expected_ch), - self.get_position_info()) - if read_ch != expected_ch: - raise_yson_error( - 'Found "{0}" while expecting "{1}" in Yson'.format(read_ch, expected_ch), - self.get_position_info()) - - def _skip_whitespaces(self): - while self._peek_char().isspace(): - self._read_char() - - def _read_string(self): - ch = self._peek_char() - if not ch: - raise_yson_error( - "Premature end-of-stream while expecting string literal in Yson", - self.get_position_info()) - if ch == STRING_MARKER: - return self._read_binary_string() - if ch == b'"': - return self._read_quoted_string() - if not ch.isalpha() and not ch == b"_" and not ch == b"%": - raise_yson_error( - "Expecting string literal but found {0} in Yson".format(ch), - self.get_position_info()) - return self._read_unquoted_string() - - def _read_binary_string(self): - self._expect_char(STRING_MARKER) - length = _zig_zag_decode(self._read_varint()) - string = self._read_binary_chars(length) - if self._output_buffer is None: - return self._decode_string(string) - - def _read_varint(self): - count = 0 - result = 0 - read_next = True - while read_next: - ch = self._read_char() - if not ch: - raise_yson_error( - "Premature end-of-stream while reading varinteger in Yson", - self.get_position_info()) - byte = ord(ch) - result |= (byte & 0x7F) << (7 * count) - if result > 2 ** 64 - 1: - raise_yson_error( - "Varinteger is too large for Int64 in Yson", - self.get_position_info()) - count += 1 - read_next = byte & 0x80 != 0 - - return yson_types._YsonIntegerBase(result) - - def _read_quoted_string(self): - self._expect_char(b'"') - if self._output_buffer is None: - result = [] - pending_next_char = False - while True: - ch = self._read_char() - if not ch: - raise_yson_error( - "Premature end-of-stream while reading string literal in Yson", - self.get_position_info()) - - if pending_next_char: - pending_next_char = False - if ch not in b"abfnrtvuUx01234567\\": - # invalid escape sequence support (pass to "unicode_escape") - result.pop() - else: - if ch == b'"': - break - if ch == b"\\": - pending_next_char = True - - if self._output_buffer is None: - result.append(ch) - if self._output_buffer is None: - return self._decode_string(self._unescape(b"".join(result))) - - def _unescape(self, string): - return string.decode("unicode_escape").encode("latin1") - - def _decode_string(self, string): - assert self._encoding is not _ENCODING_SENTINEL - if self._encoding is not None: - try: - return string.decode(self._encoding) - except UnicodeDecodeError: - proxy = yson_types.YsonStringProxy() - proxy._bytes = string - return proxy - else: - return string - - def _read_unquoted_string(self): - if self._output_buffer is None: - result = [] - while True: - ch = self._peek_char() - if ch and (ch.isalpha() or ch.isdigit() or ch in b"_%-."): - self._read_char() - if self._output_buffer is None: - result.append(ch) - else: - break - if self._output_buffer is None: - return self._decode_string(b"".join(result)) - - def _read_numeric(self): - result = [] - while True: - ch = self._peek_char() - if not ch or not (ch.isdigit() or ch in b"+-.eEu"): - break - self._read_char() - result.append(ch) - if not result: - raise_yson_error( - "Premature end-of-stream while parsing numeric literal in Yson", - self.get_position_info()) - return b"".join(result) - - def _parse_percent_literal(self): - def raise_unexpected(string): - expected = [b"%" + literal for literal in PERCENT_LITERALS] - raise_yson_error( - "Incorrect percent-preceded literal %s, expected one of %s" % (b"%" + string, expected), - self.get_position_info()) - self._expect_char(b"%") - ch = self._peek_char() - if ch not in PERCENT_LITERAL_LENGTH: - raise_unexpected(ch) - string = self._read_binary_chars(PERCENT_LITERAL_LENGTH[ch]) - if string == b"true": - return True, TOKEN_BOOLEAN - elif string == b"false": - return False, TOKEN_BOOLEAN - elif string in PERCENT_LITERALS: - return float(string), TOKEN_DOUBLE - else: - raise_unexpected(string) - - def _parse_entity(self): - self._expect_char(b"#") - return None - - def _parse_string(self): - return self._read_string() - - def _parse_binary_int64(self): - self._expect_char(INT64_MARKER) - varint = self._read_varint() - if self._output_buffer is None: - return _zig_zag_decode(varint) - - def _parse_binary_uint64(self): - self._expect_char(UINT64_MARKER) - varint = self._read_varint() - if self._output_buffer is None: - return yson_types.YsonUint64(varint) - - def _parse_binary_double(self): - self._expect_char(DOUBLE_MARKER) - bytes_ = self._read_binary_chars(struct.calcsize(b"<d")) - if self._output_buffer is None: - return struct.unpack(b"<d", bytes_)[0] - - def _parse_numeric(self): - string = self._read_numeric() - numeric_type = _get_numeric_type(string) - if numeric_type == _SEEMS_INT64: - try: - result = yson_types._YsonIntegerBase(string) - token_type = TOKEN_INT64 - if result > 2 ** 63 - 1 or result < -(2 ** 63): - raise ValueError() - except ValueError: - raise_yson_error( - "Failed to parse Int64 literal {0} in Yson".format(string), - self.get_position_info()) - elif numeric_type == _SEEMS_UINT64: - try: - if string.endswith(b"u"): - string = string[:-1] - else: - raise ValueError() - result = yson_types.YsonUint64(int(string)) - token_type = TOKEN_UINT64 - if result > 2 ** 64 - 1: - raise ValueError() - except ValueError: - raise_yson_error( - "Failed to parse Uint64 literal {0} in Yson".format(string), - self.get_position_info()) - else: - try: - result = float(string) - token_type = TOKEN_DOUBLE - except ValueError: - raise_yson_error( - "Failed to parse Double literal {0} in Yson".format(string), - self.get_position_info()) - return result, token_type diff --git a/yt/python/yt/yson/parser.py b/yt/python/yt/yson/parser.py deleted file mode 100644 index fd545a65ad7..00000000000 --- a/yt/python/yt/yson/parser.py +++ /dev/null @@ -1,279 +0,0 @@ -from . import convert -from .common import raise_yson_error, YsonError, StreamWrap, _ENCODING_SENTINEL -from .tokenizer import YsonTokenizer -from .yson_token import ( - TOKEN_STRING, - TOKEN_INT64, - TOKEN_UINT64, - TOKEN_DOUBLE, - TOKEN_BOOLEAN, - TOKEN_HASH, - TOKEN_SEMICOLON, - TOKEN_LEFT_ANGLE, - TOKEN_EQUALS, - TOKEN_RIGHT_ANGLE, - TOKEN_LEFT_BRACKET, - TOKEN_RIGHT_BRACKET, - TOKEN_LEFT_BRACE, - TOKEN_RIGHT_BRACE, - TOKEN_START_OF_STREAM, - TOKEN_END_OF_STREAM, -) - -from io import BytesIO - - -def _is_text_reader(stream): - return type(stream.read(0)) is str - - -class YsonParser(object): - def __init__(self, stream, encoding, always_create_attributes): - # COMPAT: Before porting YSON to Python 3 it supported parsing from - # unicode strings. - if _is_text_reader(stream): - raise TypeError("Only binary streams are supported by YSON parser") - self._tokenizer = YsonTokenizer(stream, encoding) - self._always_create_attributes = always_create_attributes - self._encoding = encoding - - def _has_attributes(self): - try: - self._tokenizer.parse_next() - except YsonError: - return False - return self._tokenizer.get_current_type() == TOKEN_LEFT_ANGLE - - def _parse_attributes(self): - self._tokenizer.get_current_token().expect_type(TOKEN_LEFT_ANGLE) - result = {} - while True: - self._tokenizer.parse_next() - if self._tokenizer.get_current_type() == TOKEN_RIGHT_ANGLE: - break - self._tokenizer.get_current_token().expect_type(TOKEN_STRING) - key = self._tokenizer.get_current_token().get_value() - if not key: - raise_yson_error( - "Empty attribute name in Yson", - self._tokenizer.get_position_info()) - self._tokenizer.parse_next() - self._tokenizer.get_current_token().expect_type(TOKEN_EQUALS) - self._tokenizer.parse_next() - value = self._parse_any() - if key in result: - raise_yson_error( - 'Repeated attribute "{0}" in Yson'.format(key), - self._tokenizer.get_position_info()) - result[key] = value - self._tokenizer.parse_next() - if self._tokenizer.get_current_type() == TOKEN_RIGHT_ANGLE: - break - self._tokenizer.get_current_token().expect_type(TOKEN_SEMICOLON) - self._tokenizer.get_current_token().expect_type(TOKEN_RIGHT_ANGLE) - return result - - def _parse_list(self): - self._tokenizer.get_current_token().expect_type(TOKEN_LEFT_BRACKET) - result = [] - while True: - self._tokenizer.parse_next() - if self._tokenizer.get_current_type() == TOKEN_RIGHT_BRACKET: - break - value = self._parse_any() - result.append(value) - self._tokenizer.parse_next() - if self._tokenizer.get_current_type() == TOKEN_RIGHT_BRACKET: - break - self._tokenizer.get_current_token().expect_type(TOKEN_SEMICOLON) - self._tokenizer.get_current_token().expect_type(TOKEN_RIGHT_BRACKET) - return result - - def _parse_map(self): - self._tokenizer.get_current_token().expect_type(TOKEN_LEFT_BRACE) - result = {} - while True: - self._tokenizer.parse_next() - if self._tokenizer.get_current_type() == TOKEN_RIGHT_BRACE: - break - self._tokenizer.get_current_token().expect_type(TOKEN_STRING) - key = self._tokenizer.get_current_token().get_value() - self._tokenizer.parse_next() - self._tokenizer.get_current_token().expect_type(TOKEN_EQUALS) - self._tokenizer.parse_next() - value = self._parse_any() - if key in result: - raise_yson_error( - 'Duplicate map key "{0}" in YSON'.format(key), - self._tokenizer.get_position_info()) - result[key] = value - self._tokenizer.parse_next() - if self._tokenizer.get_current_type() == TOKEN_RIGHT_BRACE: - break - self._tokenizer.get_current_token().expect_type(TOKEN_SEMICOLON) - self._tokenizer.get_current_token().expect_type(TOKEN_RIGHT_BRACE) - return result - - def _parse_any(self): - if self._tokenizer.get_current_type() == TOKEN_START_OF_STREAM: - self._tokenizer.parse_next() - attributes = None - if self._tokenizer.get_current_type() == TOKEN_LEFT_ANGLE: - attributes = self._parse_attributes() - self._tokenizer.parse_next() - - if self._tokenizer.get_current_type() == TOKEN_END_OF_STREAM: - raise_yson_error( - "Premature end-of-stream in Yson", - self._tokenizer.get_position_info()) - - if self._tokenizer.get_current_type() == TOKEN_LEFT_BRACKET: - result = self._parse_list() - - elif self._tokenizer.get_current_type() == TOKEN_LEFT_BRACE: - result = self._parse_map() - - elif self._tokenizer.get_current_type() == TOKEN_HASH: - result = None - - else: - self._tokenizer.get_current_token().expect_type((TOKEN_BOOLEAN, TOKEN_INT64, TOKEN_UINT64, - TOKEN_STRING, TOKEN_DOUBLE)) - result = self._tokenizer.get_current_token().get_value() - - return convert.to_yson_type( - result, - attributes=attributes, - always_create_attributes=self._always_create_attributes, - encoding=self._encoding, - ) - - def parse(self): - result = self._parse_any() - self._tokenizer.parse_next() - self._tokenizer.get_current_token().expect_type(TOKEN_END_OF_STREAM) - return result - - -class RawYsonParser(object): - def __init__(self, stream): - if _is_text_reader(stream): - raise TypeError("Only binary streams are supported by YSON parser") - self._buffer = bytearray() - self._tokenizer = YsonTokenizer(stream, output_buffer=self._buffer) - - def _parse_mapping(self, end_token): - while True: - self._tokenizer.parse_next() - if self._tokenizer.get_current_type() == end_token: - break - self._tokenizer.get_current_token().expect_type(TOKEN_STRING) - self._tokenizer.parse_next() - self._tokenizer.get_current_token().expect_type(TOKEN_EQUALS) - self._tokenizer.parse_next() - self._parse_any() - self._tokenizer.parse_next() - if self._tokenizer.get_current_type() == end_token: - break - self._tokenizer.get_current_token().expect_type(TOKEN_SEMICOLON) - self._tokenizer.get_current_token().expect_type(end_token) - - def _parse_attributes(self): - self._tokenizer.get_current_token().expect_type(TOKEN_LEFT_ANGLE) - self._parse_mapping(TOKEN_RIGHT_ANGLE) - - def _parse_map(self): - self._tokenizer.get_current_token().expect_type(TOKEN_LEFT_BRACE) - self._parse_mapping(TOKEN_RIGHT_BRACE) - - def _parse_list(self): - self._tokenizer.get_current_token().expect_type(TOKEN_LEFT_BRACKET) - while True: - self._tokenizer.parse_next() - if self._tokenizer.get_current_type() == TOKEN_RIGHT_BRACKET: - break - self._parse_any() - self._tokenizer.parse_next() - if self._tokenizer.get_current_type() == TOKEN_RIGHT_BRACKET: - break - self._tokenizer.get_current_token().expect_type(TOKEN_SEMICOLON) - self._tokenizer.get_current_token().expect_type(TOKEN_RIGHT_BRACKET) - - def _parse_any(self): - if self._tokenizer.get_current_type() == TOKEN_START_OF_STREAM: - self._tokenizer.parse_next() - - if self._tokenizer.get_current_type() == TOKEN_LEFT_ANGLE: - self._parse_attributes() - self._tokenizer.parse_next() - - if self._tokenizer.get_current_type() == TOKEN_END_OF_STREAM: - raise_yson_error( - "Premature end-of-stream in Yson", - self._tokenizer.get_position_info()) - - if self._tokenizer.get_current_type() == TOKEN_LEFT_BRACKET: - self._parse_list() - - elif self._tokenizer.get_current_type() == TOKEN_LEFT_BRACE: - self._parse_map() - - elif self._tokenizer.get_current_type() == TOKEN_HASH: - pass - - else: - self._tokenizer.get_current_token().expect_type((TOKEN_BOOLEAN, TOKEN_INT64, TOKEN_UINT64, - TOKEN_STRING, TOKEN_DOUBLE)) - - def _flush_buffer(self): - res = bytes(self._buffer) - self._buffer[:] = b'' - return res - - def parse(self): - while self._tokenizer.get_current_type() != TOKEN_END_OF_STREAM: - self._parse_any() - self._tokenizer.parse_next() - self._tokenizer.get_current_token().expect_type(TOKEN_SEMICOLON) - yield self._flush_buffer() - self._tokenizer.parse_next() - - -def load(stream, yson_type=None, always_create_attributes=True, raw=None, - encoding=_ENCODING_SENTINEL, lazy=False): - """Deserializes object from YSON formatted stream `stream`. - - :param str yson_type: type of YSON, one of ["node", "list_fragment", "map_fragment"]. - """ - if lazy: - raise YsonError("Lazy parsing is not supported in python parser") - - if raw: - if yson_type != "list_fragment": - raise YsonError("Raw mode is only supported for list fragments") - return RawYsonParser(stream).parse() - - if encoding is _ENCODING_SENTINEL: - encoding = "utf-8" - - if yson_type == "list_fragment": - stream = StreamWrap(stream, b"[", b"]") - elif yson_type == "map_fragment": - stream = StreamWrap(stream, b"{", b"}") - else: - if yson_type is not None: - raise YsonError("Unexpected yson type: {0!r}".format(yson_type)) - - parser = YsonParser(stream, encoding, always_create_attributes) - return parser.parse() - - -def loads(string, yson_type=None, always_create_attributes=True, raw=None, - encoding=_ENCODING_SENTINEL, lazy=False): - """Deserializes object from YSON formatted string `string`. See :func:`load <.load>`.""" - if type(string) is str: - string = string.encode(encoding) - raise TypeError("Only binary streams are supported by YSON parser") - return load(BytesIO(string), yson_type=yson_type, - always_create_attributes=always_create_attributes, - raw=raw, encoding=encoding, lazy=lazy) diff --git a/yt/python/yt/yson/tokenizer.py b/yt/python/yt/yson/tokenizer.py deleted file mode 100644 index eb4d5a77a92..00000000000 --- a/yt/python/yt/yson/tokenizer.py +++ /dev/null @@ -1,21 +0,0 @@ -from .yson_token import YsonToken, TOKEN_START_OF_STREAM -from .lexer import YsonLexer -from .common import _ENCODING_SENTINEL - - -class YsonTokenizer(object): - def __init__(self, input_str, encoding=_ENCODING_SENTINEL, output_buffer=None): - self._token = YsonToken(type=TOKEN_START_OF_STREAM) - self._lexer = YsonLexer(input_str, encoding=encoding, output_buffer=output_buffer) - - def parse_next(self): - self._token = self._lexer.get_next_token() - - def get_current_token(self): - return self._token - - def get_current_type(self): - return self.get_current_token().get_type() - - def get_position_info(self): - return self._lexer.get_position_info() diff --git a/yt/python/yt/yson/writer.py b/yt/python/yt/yson/writer.py deleted file mode 100644 index 4fc16e001f0..00000000000 --- a/yt/python/yt/yson/writer.py +++ /dev/null @@ -1,431 +0,0 @@ -# -*- coding: utf-8 -*- - -"""``yson`` exposes an API familiar to users of the standard library -:mod:`marshal` and :mod:`pickle` modules. - -Serializable types (rules applied top to bottom): -int, long -> yson int -str -> yson string -unicode -> yson string (using specified encoding) -any Mapping -> yson dict -any Iterable -> yson list - -Simple examples:: - - >>> import yson - >>> b = [4, 5, 6] - >>> print yson.dumps({"a" : b, "b" : b}, indent=" ") - { - "a" : [ - 4; - 5; - 6; - ]; - "b" : [ - 4; - 5; - 6; - ]; - } - >>> print yson.dumps(("a", "a"), indent=" ") - [ - "a"; - "a"; - ] - >>> print yson.dumps(123456) - 123456 - >>> print yson.dumps(u'"Hello world!" -- "Превед, медвед!"') - "\"Hello world!\" -- \"\xd0\x9f\xd1\x80\xd0\xb5\xd0\xb2\xd0\xb5\xd0\xb4, \xd0\xbc\xd0\xb5\xd0\xb4\xd0\xb2\xd0\xb5\xd0\xb4!\"" -""" - -from .common import (YsonError, - STRING_MARKER, INT64_MARKER, DOUBLE_MARKER, - FALSE_MARKER, TRUE_MARKER, UINT64_MARKER) -from . import yson_types - -import math -import struct -# Python3 compatibility -try: - from collections.abc import Iterable, Mapping -except ImportError: - from collections import Iterable, Mapping - -__all__ = ["dump", "dumps"] - - -def _is_hex_digit(c): - return ord(b'0') <= c <= ord(b'9') or ord(b'A') <= c <= ord(b'F') or ord(b'a') <= c <= ord(b'f') - - -def _is_oct_digit(c): - return ord(b'0') <= c <= ord(b'7') - - -def _escape_byte(c, nxt, r): - if c == ord(b"\""): - r += b"\\\"" - elif c == ord(b"\\"): - r += b"\\\\" - elif 32 <= c <= 126: - r.append(c) - elif c == ord(b"\r"): - r += b"\\r" - elif c == ord(b"\n"): - r += b"\\n" - elif c == ord(b"\t"): - r += b"\\t" - elif c < 8 and not _is_oct_digit(nxt): - r += '\\{}'.format(c).encode("ascii") - elif not _is_hex_digit(nxt): - r += '\\x{:02X}'.format(c).encode("ascii") - else: - r += '\\{:03o}'.format(c).encode("ascii") - - -def _escape_bytes(obj): - if len(obj) == 0: - return b"" - - res = bytearray() - iterator = iter(obj) - cur = next(iterator) - for nxt in iterator: - _escape_byte(cur, nxt, res) - cur = nxt - _escape_byte(cur, ord(b" "), res) - return bytes(res) - - -def dump(object, stream, yson_format=None, yson_type=None, indent=None, - ignore_inner_attributes=False, encoding="utf-8", sort_keys=False, - check_circular=True): - """Serializes `object` as a YSON formatted stream to `stream`. - - :param str yson_format: format of YSON, one of ["binary", "text", "pretty"]. - :param str yson_type: type of YSON, one of ["node", "list_fragment", "map_fragment"]. - :param int indent: number of indentation spaces in pretty format. - :param bool ignore_inner_attributes: skip attributes of non-top-level values. - :param str encoding: encoding that uses to encode unicode strings. - :param bool sort_keys: if True, mapping items are printed in sorted order. - :param bool check_circular: prevent the attempt to serialize an object with reference loop. - """ - - stream.write(dumps(object, yson_format=yson_format, yson_type=yson_type, indent=indent, - ignore_inner_attributes=ignore_inner_attributes, - encoding=encoding, sort_keys=sort_keys, - check_circular=check_circular)) - - -class YsonContext(object): - def __init__(self): - self.path_parts = [] - self.row_index = None - - def push(self, key_or_index): - self.path_parts.append(key_or_index) - - def pop(self): - self.path_parts.pop() - - -def _raise_error_with_context(message, context): - attributes = {} - if context.row_index is not None: - attributes["row_index"] = context.row_index - - path_parts = map(str, context.path_parts) - if context.path_parts: - attributes["row_key_path"] = "/" + "/".join(path_parts) - raise YsonError(message, attributes=attributes) - - -def _zig_zag_encode(value): - return (value >> 63) ^ (value << 1) - - -def _dump_varint(value): - assert 0 <= value <= 2 ** 64 - 1 - result = bytearray() - while value >= 0x80: - result.append(0x80 | (value & 0x7F)) - value >>= 7 - result.append(value) - return bytes(result) - - -def dumps(object, yson_format=None, yson_type=None, indent=None, - ignore_inner_attributes=False, encoding="utf-8", sort_keys=False, - check_circular=True): - """Serializes `object` as a YSON formatted stream to string and returns it. See :func:`dump <.dump>`.""" - if indent is None: - indent = 4 - if isinstance(indent, int): - indent = b" " * indent - if yson_format is None: - yson_format = "text" - if yson_format not in ("pretty", "text", "binary"): - raise YsonError("{0} format is not supported".format(yson_format)) - if yson_format in ("text", "binary"): - indent = None - if yson_type is not None: - if yson_type not in ["list_fragment", "map_fragment", "node"]: - raise YsonError("YSON type {0} is not supported".format(yson_type)) - else: - yson_type = "node" - - is_text = yson_format in ("text", "pretty") - d = Dumper(check_circular, encoding, indent, yson_type, sort_keys, - is_text=is_text, ignore_inner_attributes=ignore_inner_attributes) - return d.dumps(object, YsonContext()) - - -class Dumper(object): - def __init__(self, check_circular, encoding, indent, yson_type, sort_keys, - is_text, ignore_inner_attributes): - self.yson_type = yson_type - - self._seen_objects = None - if check_circular: - self._seen_objects = {} - - self._encoding = encoding - self._format = FormatDetails(indent, sort_keys) - if yson_type == "node": - self._level = -1 - else: - self._level = -2 # Stream elements are one level deep, but need not be indented - - self._is_text = is_text - self._ignore_inner_attributes = ignore_inner_attributes - - def _has_attributes(self, obj): - if hasattr(obj, "has_attributes"): - return obj.has_attributes() - return hasattr(obj, "attributes") - - def dumps(self, obj, context): - if hasattr(obj, "to_yson_type") and callable(obj.to_yson_type): - return self.dumps(obj.to_yson_type(), context) - self._level += 1 - attributes = b"" - if self._has_attributes(obj) and (not self._ignore_inner_attributes or self._level == 0): - if not isinstance(obj.attributes, dict): - _raise_error_with_context('Invalid field "attributes": it must be string or None', context) - if obj.attributes: - attributes = self._dump_attributes(obj.attributes, context) - - result = None - if obj is False or (isinstance(obj, yson_types.YsonBoolean) and not obj): - if self._is_text: - result = b"%false" - else: - result = FALSE_MARKER - elif obj is True or (isinstance(obj, yson_types.YsonBoolean) and obj): - if self._is_text: - result = b"%true" - else: - result = TRUE_MARKER - elif isinstance(obj, int): - if obj < -2 ** 63 or obj >= 2 ** 64: - _raise_error_with_context("Integer {0} cannot be represented in YSON " - "since it is out of range [-2^63, 2^64 - 1])".format(obj), context) - - greater_than_max_int64 = obj >= 2 ** 63 - if isinstance(obj, yson_types.YsonUint64) and obj < 0: - _raise_error_with_context("Can not dump negative integer as YSON uint64", context) - if isinstance(obj, yson_types.YsonInt64) and greater_than_max_int64: - _raise_error_with_context("Can not dump integer greater than 2^63-1 as YSON int64", context) - - result = self._dump_integer(obj, greater_than_max_int64) - elif isinstance(obj, float): - result = self._dump_float(obj) - elif isinstance(obj, (str, bytes, yson_types.YsonStringProxy)): - result = self._dump_string(obj, context) - elif isinstance(obj, Mapping): - result = self._dump_map(obj, context) - elif isinstance(obj, Iterable): - result = self._dump_list(obj, context) - elif isinstance(obj, yson_types.YsonEntity) or obj is None: - result = b"#" - else: - _raise_error_with_context("{0!r} is not Yson serializable".format(obj), context) - self._level -= 1 - return attributes + result - - def _dump_integer(self, obj, force_uint64): - if self._is_text: - if isinstance(obj, (yson_types.YsonInt64, yson_types.YsonUint64)): - obj_str = str(yson_types._YsonIntegerBase(obj)) - else: - obj_str = str(obj) - - result = obj_str.encode("ascii") - if force_uint64 or isinstance(obj, yson_types.YsonUint64): - result += b"u" - else: - if force_uint64 or isinstance(obj, yson_types.YsonUint64): - result = UINT64_MARKER + _dump_varint(obj) - else: - result = INT64_MARKER + _dump_varint(_zig_zag_encode(obj)) - return result - - def _dump_float(self, obj): - if self._is_text: - if math.isnan(obj): - result = b"%nan" - elif math.isinf(obj): - if obj > 0: - result = b"%inf" - else: - result = b"%-inf" - else: - if type(obj) is yson_types.YsonDouble: - obj_str = str(float(obj)) - else: - obj_str = str(obj) - result = obj_str.encode("ascii") - else: - result = DOUBLE_MARKER + struct.pack("<d", obj) - return result - - def _dump_string(self, obj, context): - if isinstance(obj, bytes): - result = obj - elif isinstance(obj, str): - if self._encoding is None: - _raise_error_with_context('Cannot encode unicode object {0!r} to bytes since "encoding" ' - 'parameter is None. Consider using byte strings ' - 'instead or specify encoding'.format(obj), - context) - result = obj.encode(self._encoding) - elif isinstance(obj, yson_types.YsonStringProxy): - result = obj._bytes - else: - assert False - if self._is_text: - return b"".join([b'"', _escape_bytes(result), b'"']) - else: - encoded_len = _zig_zag_encode(len(result)) - return b"".join([STRING_MARKER, _dump_varint(encoded_len), result]) - - def _dump_map(self, obj, context): - is_stream = self.yson_type == "map_fragment" and self._level == -1 - result = [] - if not is_stream: - result += [b"{", self._format.nextline()] - - for k, v in self._format.mapping_iter(obj): - if not isinstance(k, (str, bytes, yson_types.YsonStringProxy)): - _raise_error_with_context("Only string can be Yson map key. Key: {0!r}".format(k), context) - - @self._circular_check(v) - def process_item(): - context.push(k) - item = [self._format.prefix(self._level + 1), - self._dump_string(k, context), self._format.space(), b"=", - self._format.space(), self.dumps(v, context), b";", self._format.nextline(is_stream)] - context.pop() - return item - - result += process_item() - - if not is_stream: - result += [self._format.prefix(self._level), b"}"] - - return b"".join(result) - - def _dump_list(self, obj, context): - is_stream = self.yson_type == "list_fragment" and self._level == -1 - result = [] - if not is_stream: - result += [b"[", self._format.nextline()] - - for index, v in enumerate(obj): - @self._circular_check(v) - def process_item(): - if is_stream: - context.row_index = index - else: - context.push(index) - item = [self._format.prefix(self._level + 1), - self.dumps(v, context), b";", self._format.nextline(is_stream)] - if not is_stream: - context.pop() - return item - - result += process_item() - - if not is_stream: - result += [self._format.prefix(self._level), b"]"] - - return b"".join(result) - - def _dump_attributes(self, obj, context): - result = [b"<", self._format.nextline()] - for k, v in obj.items(): - if not isinstance(k, (str, bytes)): - _raise_error_with_context("Only string can be Yson map key. Key: {0!r}".format(obj), context) - - @self._circular_check(v) - def process_item(): - context.push("@" + k) - item = [self._format.prefix(self._level + 1), - self._dump_string(k, context), self._format.space(), b"=", - self._format.space(), self.dumps(v, context), b";", self._format.nextline()] - context.pop() - return item - - result += process_item() - result += [self._format.prefix(self._level), b">"] - return b"".join(result) - - def _circular_check(self, obj): - def decorator(fn): - def wrapper(*args, **kwargs): - obj_id = None - if self._seen_objects is not None: - obj_id = id(obj) - if obj_id in self._seen_objects: - raise YsonError("Circular reference detected. Object: {0!r}".format(obj)) - else: - self._seen_objects[obj_id] = obj - - result = fn(*args, **kwargs) - - if self._seen_objects: - del self._seen_objects[obj_id] - return result - - return wrapper - return decorator - - -class FormatDetails(object): - def __init__(self, indent, sort_keys=False): - self._indent = indent - self._sort_keys = sort_keys - - def prefix(self, level): - if self._indent: - return b"".join([self._indent] * level) - else: - return b"" - - def nextline(self, force=False): - if force or self._indent: - return b"\n" - else: - return b"" - - def space(self): - if self._indent: - return b" " - else: - return b"" - - def mapping_iter(self, mapping): - if self._sort_keys: - return ((key, mapping[key]) for key in sorted(mapping.keys())) - else: - return mapping.items() diff --git a/yt/python/yt/yson/ya.make b/yt/python/yt/yson/ya.make deleted file mode 100644 index 5a8a64ea6bc..00000000000 --- a/yt/python/yt/yson/ya.make +++ /dev/null @@ -1,27 +0,0 @@ -PY23_LIBRARY() - -INCLUDE(${ARCADIA_ROOT}/yt/opensource.inc) - -IF (PYTHON2) - PEERDIR(yt/python_py2/yt/yson) -ELSE() - PEERDIR( - yt/python/yt - ) - - PY_SRCS( - NAMESPACE yt.yson - - __init__.py - common.py - convert.py - lexer.py - parser.py - tokenizer.py - writer.py - yson_token.py - yson_types.py - ) -ENDIF() - -END() diff --git a/yt/python/yt/yson/yson_token.py b/yt/python/yt/yson/yson_token.py deleted file mode 100644 index 43caa0a673f..00000000000 --- a/yt/python/yt/yson/yson_token.py +++ /dev/null @@ -1,146 +0,0 @@ -from .common import YsonError - -from yt.common import flatten - -import string - - -TOKEN_LITERAL = 0 -TOKEN_SLASH = 1 -TOKEN_AMPERSAND = 2 -TOKEN_AT = 3 -TOKEN_ASTERISK = 4 -TOKEN_START_OF_STREAM = 5 -TOKEN_END_OF_STREAM = 6 -TOKEN_RANGE = 7 -TOKEN_SEMICOLON = 8 -TOKEN_EQUALS = 9 -TOKEN_LEFT_BRACE = 10 -TOKEN_RIGHT_BRACE = 11 -TOKEN_HASH = 12 -TOKEN_LEFT_BRACKET = 13 -TOKEN_RIGHT_BRACKET = 14 -TOKEN_LEFT_ANGLE = 15 -TOKEN_RIGHT_ANGLE = 16 -TOKEN_LEFT_PARENTHESIS = 17 -TOKEN_RIGHT_PARENTHESIS = 18 -TOKEN_COLON = 19 -TOKEN_COMMA = 20 -TOKEN_STRING = 21 -TOKEN_INT64 = 22 -TOKEN_UINT64 = 23 -TOKEN_DOUBLE = 24 -TOKEN_BOOLEAN = 25 -TOKEN_SPECIAL = 26 - -CHAR_TO_TOKEN_TYPE = { - ";": TOKEN_SEMICOLON, - "=": TOKEN_EQUALS, - "{": TOKEN_LEFT_BRACE, - "}": TOKEN_RIGHT_BRACE, - "#": TOKEN_HASH, - "[": TOKEN_LEFT_BRACKET, - "]": TOKEN_RIGHT_BRACKET, - "<": TOKEN_LEFT_ANGLE, - ">": TOKEN_RIGHT_ANGLE, - "(": TOKEN_LEFT_PARENTHESIS, - ")": TOKEN_RIGHT_PARENTHESIS, - ":": TOKEN_COLON, - ",": TOKEN_COMMA, - "/": TOKEN_SLASH, - "@": TOKEN_AT, - "&": TOKEN_AMPERSAND, - "*": TOKEN_ASTERISK -} - - -def char_to_token_type(char_or_byte): - char_or_byte = chr(char_or_byte) - if char_or_byte not in CHAR_TO_TOKEN_TYPE: - return TOKEN_END_OF_STREAM - return CHAR_TO_TOKEN_TYPE[char_or_byte] - - -def token_type_to_string(token): - names = { - TOKEN_LITERAL: "Literal", - TOKEN_SLASH: "Slash", - TOKEN_AMPERSAND: "Ampersand", - TOKEN_AT: "At", - TOKEN_ASTERISK: "Asterisk", - TOKEN_START_OF_STREAM: "Start-of-stream", - TOKEN_END_OF_STREAM: "End-of-stream", - TOKEN_RANGE: "Range", - TOKEN_SEMICOLON: "Semicolon", - TOKEN_EQUALS: "Equals", - TOKEN_LEFT_BRACE: "Left-brace", - TOKEN_RIGHT_BRACE: "Right-brace", - TOKEN_HASH: "Hash", - TOKEN_LEFT_BRACKET: "Left-bracket", - TOKEN_RIGHT_BRACKET: "Right-bracket", - TOKEN_LEFT_ANGLE: "Left-angle", - TOKEN_RIGHT_ANGLE: "Right-angle", - TOKEN_LEFT_PARENTHESIS: "Left-parenthesis", - TOKEN_RIGHT_PARENTHESIS: "Right-parenthesis", - TOKEN_COLON: "Colon", - TOKEN_COMMA: "Comma", - TOKEN_STRING: "String", - TOKEN_INT64: "Int64", - TOKEN_UINT64: "Uint64", - TOKEN_DOUBLE: "Double", - TOKEN_BOOLEAN: "Boolean", - TOKEN_SPECIAL: "Special", - } - if token is None: - return "Unknown" - return names[token] - - -def decode_token_value(value): - if not isinstance(value, bytes): - return value - chars = [] - for byte in value: - char = chr(byte) - if char in string.printable: # whitespaces cannot present in token - chars.append(char) - else: - chars.append("\\x" + hex(byte)[2:]) - - return "".join(chars) - - -class YsonToken(object): - def __init__(self, value="", type=TOKEN_END_OF_STREAM): - self._value = value - self._type = type - - def get_value(self): - return self._value - - def get_type(self): - return self._type - - def _raise_error(self, message_end_of_stream, message_unexpected_token, token_type, value, expected_type): - if token_type == TOKEN_END_OF_STREAM: - raise YsonError(message_end_of_stream.format(expected_type)) - else: - raise YsonError(message_unexpected_token.format(value, token_type_to_string(token_type), expected_type)) - - def expect_type(self, type_or_types): - token_type = self.get_type() - expected_types = flatten(type_or_types) - if token_type is None: - raise YsonError('Unexpected "{0}" while parsing node'.format(decode_token_value(self.get_value()))) - - if token_type not in expected_types: - if token_type == TOKEN_END_OF_STREAM: - raise YsonError("Unexpected end of stream; expected types are {0}".format(list(map(token_type_to_string, expected_types)))) - else: - raise YsonError('Unexpected token "{0}" of type {1}; ' - 'expected types are {2}'.format(decode_token_value(self.get_value()), - token_type_to_string(token_type), - list(map(token_type_to_string, expected_types)))) - - def __str__(self): - return str(self._value) diff --git a/yt/python/yt/yson/yson_types.py b/yt/python/yt/yson/yson_types.py deleted file mode 100644 index ed21092f7c6..00000000000 --- a/yt/python/yt/yson/yson_types.py +++ /dev/null @@ -1,343 +0,0 @@ -from yt.common import YtError - - -class YsonType(object): - def __getattr__(self, attribute): - if attribute == "attributes": - self.__dict__[attribute] = {} - return self.__dict__[attribute] - raise AttributeError('Attribute "{0}" not found'.format(attribute)) - - def has_attributes(self): - try: - return "attributes" in self.__dict__ and self.attributes is not None and self.attributes != {} - except: # noqa - return False - - def __eq__(self, other): - try: - has_attributes = other.has_attributes() - except AttributeError: - has_attributes = False - if has_attributes: - return self.attributes == other.attributes - return not self.has_attributes() - - def __ne__(self, other): - return not (self == other) - - def to_str(self, base_type, str_func): - if self.has_attributes(): - return str_func({"value": base_type(self), "attributes": self.attributes}) - return str_func(base_type(self)) - - def base_hash(self, type_): - if self.has_attributes(): - raise TypeError("unhashable type: YSON has non-trivial attributes") - return hash(type_(self)) - - def get_yson_type_str(selfself): - return None - - -class YsonString(bytes, YsonType): - def __eq__(self, other): - # COMPAT: With implicit promotion of str to unicode it can make sense - # to compare binary YsonString to unicode string. - if not isinstance(other, (bytes, str)): - return NotImplemented - return bytes(self) == bytes(other) and YsonType.__eq__(self, other) - - def __ne__(self, other): - return not (self == other) - - def __hash__(self): - return self.base_hash(bytes) - - def __repr__(self): - return self.to_str(bytes, repr) - - def get_yson_type_str(self): - return "string" - - -class YsonUnicode(str, YsonType): - def __eq__(self, other): - if not isinstance(other, str): - return NotImplemented - return str(self) == str(other) and YsonType.__eq__(self, other) - - def __ne__(self, other): - return not (self == other) - - def __hash__(self): - return self.base_hash(str) - - def __repr__(self): - return self.to_str(str, repr) - - def get_yson_type_str(self): - return "string" - - -class NotUnicodeError(YtError, TypeError): - pass - - -def _truncate(s, length=50): - assert isinstance(s, bytes) - if len(s) < length: - return s - return s[:length] + b"..." - - -def _make_raise_not_unicode_error(name): - def fun(self, *args, **kwargs): - raise NotUnicodeError('Method "{}" is not allowed: YSON string "{}" ' - "could not be decoded to Unicode, " - "see https://ytsaurus.tech/docs/en/api/python/userdoc#python3_strings" - .format(name, _truncate(self._bytes))) - return fun - - -def proxy(cls): - ALLOWED_METHODS = [ - "get_bytes", - "is_unicode", - "__hash__", - "__eq__", - "__ne__", - "__repr__", - "__format__", - "__dict__", - "__qualname__", - "__class__", - "__mro__", - "__new__", - "__init__", - "__getattr__", - "__setattr__", - "__getattribute__", - "__copy__", - "__deepcopy__", - ] - - ADDITIONAL_METHODS = [ - "__radd__", - ] - - for name in dir(str): - attr = getattr(str, name) - if callable(attr) and name not in ALLOWED_METHODS: - setattr(cls, name, _make_raise_not_unicode_error(name)) - for name in ADDITIONAL_METHODS: - setattr(cls, name, _make_raise_not_unicode_error(name)) - return cls - - -# NB: This class is never returned by library in Python2. -# NB: Don't create this class by hand, it should only be returned -# from the library. -@proxy -class YsonStringProxy(YsonType): - def __repr__(self): - value = "<YsonStringProxy>{!r}".format(self._bytes) - if self.has_attributes(): - return repr({"attributes": self.attributes, "value": value}) - return value - - def __format__(self, format_spec): - return repr(self) - - def __copy__(self): - return self - - def __deepcopy__(self, memo): - return self - - def __hash__(self): - return hash(self._bytes) - - def __eq__(self, other): - if isinstance(other, bytes): - return self._bytes == bytes(other) and YsonType.__eq__(self, other) - elif isinstance(other, YsonStringProxy): - return self._bytes == other._bytes and YsonType.__eq__(self, other) - else: - return NotImplemented - - def __ne__(self, other): - return not (self == other) - - def get_yson_type_str(self): - return "string" - - -def is_unicode(x): - return isinstance(x, str) - - -def get_bytes(x, encoding="utf8"): - if isinstance(x, str): - return x.encode(encoding) - elif isinstance(x, YsonStringProxy): - return x._bytes - elif isinstance(x, bytes): - return x - else: - raise TypeError("get_bytes() expected str, bytes or YsonStringProxy, got <{}>{!r}" - .format(type(x), x)) - - -def make_byte_key(s): - proxy = YsonStringProxy() - proxy._bytes = s - return proxy - - -_YsonIntegerBase = int - - -class YsonIntegerBase(_YsonIntegerBase, YsonType): - def __eq__(self, other): - if not isinstance(other, int): - return NotImplemented - return _YsonIntegerBase(self) == _YsonIntegerBase(other) and YsonType.__eq__(self, other) - - def __ne__(self, other): - return not (self == other) - - def __hash__(self): - return self.base_hash(_YsonIntegerBase) - - def __repr__(self): - return self.to_str(_YsonIntegerBase, repr) - - def __str__(self): - return self.to_str(_YsonIntegerBase, str) - - -class YsonInt64(YsonIntegerBase): - def get_yson_type_str(self): - return "int64" - - -class YsonUint64(YsonIntegerBase): - def get_yson_type_str(self): - return "uint64" - - -class YsonDouble(float, YsonType): - def __eq__(self, other): - if not isinstance(other, float): - return NotImplemented - return float(self) == float(other) and YsonType.__eq__(self, other) - - def __ne__(self, other): - return not (self == other) - - def __hash__(self): - return self.base_hash(float) - - def __repr__(self): - return self.to_str(float, repr) - - def __str__(self): - return self.to_str(float, str) - - def get_yson_type_str(self): - return "double" - - -class YsonBoolean(int, YsonType): - def __eq__(self, other): - if not isinstance(other, int): - return NotImplemented - return (int(self) == 0) == (int(other) == 0) and YsonType.__eq__(self, other) - - def __ne__(self, other): - return not (self == other) - - def __hash__(self): - return self.base_hash(bool) - - # NB: do not change this representation, because - # this type required to be JSON serializable. - # JSON encoder thinks that it is integer and calls str. - def __repr__(self): - return "true" if self else "false" - - def __str__(self): - return self.__repr__() - - def get_yson_type_str(self): - return "bool" - - -class YsonList(list, YsonType): - def __eq__(self, other): - if not isinstance(other, list): - return NotImplemented - return list(self) == list(other) and YsonType.__eq__(self, other) - - def __ne__(self, other): - return not (self == other) - - def __hash__(self): - raise TypeError('unhashable type "YsonList"') - - def __repr__(self): - return self.to_str(list, repr) - - def __str__(self): - return self.to_str(list, str) - - -class YsonMap(dict, YsonType): - def __eq__(self, other): - if not isinstance(other, dict): - return NotImplemented - return dict(self) == dict(other) and YsonType.__eq__(self, other) - - def __ne__(self, other): - return not (self == other) - - def __hash__(self): - raise TypeError('unhashable type "YsonMap"') - - def __repr__(self): - return self.to_str(dict, repr) - - def __str__(self): - return self.to_str(dict, str) - - -class YsonEntity(YsonType): - def __init__(self, value=None): - if value is not None: - assert isinstance(value, YsonEntity) - self.attributes = value.attributes - - def __eq__(self, other): - if other is None and not self.attributes: - return True - if not isinstance(other, YsonEntity): - return NotImplemented - return YsonType.__eq__(self, other) - - def __ne__(self, other): - return not (self == other) - - def __bool__(self): - return False - - def __repr__(self): - if self.attributes: - return repr({"value": "YsonEntity", "attributes": self.attributes}) - else: - return "YsonEntity" - - def __str__(self): - return self.__repr__() - - __nonzero__ = __bool__ |