diff options
author | robot-contrib <robot-contrib@yandex-team.ru> | 2022-05-11 15:33:00 +0300 |
---|---|---|
committer | robot-contrib <robot-contrib@yandex-team.ru> | 2022-05-11 15:33:00 +0300 |
commit | 08933c12d88539cf05b655bf2a09aeb1d0aeb3e9 (patch) | |
tree | fa60816e749b2b704a7e06fd15af4207c7944abc | |
parent | 870d76c3b525ec595b1f4576275d48be9c9e513c (diff) | |
download | ydb-08933c12d88539cf05b655bf2a09aeb1d0aeb3e9.tar.gz |
Update contrib/python/xmltodict/py3 to 0.13.0
ref:e4cf17d2224cdf02408f113551e2127c0db6a8d0
-rw-r--r-- | contrib/python/xmltodict/py3/.dist-info/METADATA | 56 | ||||
-rw-r--r-- | contrib/python/xmltodict/py3/README.md | 45 | ||||
-rw-r--r-- | contrib/python/xmltodict/py3/tests/test_dicttoxml.py | 8 | ||||
-rw-r--r-- | contrib/python/xmltodict/py3/tests/test_xmltodict.py | 103 | ||||
-rw-r--r-- | contrib/python/xmltodict/py3/xmltodict.py | 96 | ||||
-rw-r--r-- | ydb/tests/functional/sqs/test_generic_messaging.py | 3 | ||||
-rw-r--r-- | ydb/tests/functional/sqs/test_queues_managing.py | 5 |
7 files changed, 271 insertions, 45 deletions
diff --git a/contrib/python/xmltodict/py3/.dist-info/METADATA b/contrib/python/xmltodict/py3/.dist-info/METADATA index 73ad4cc22f..e2aaa3e797 100644 --- a/contrib/python/xmltodict/py3/.dist-info/METADATA +++ b/contrib/python/xmltodict/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: xmltodict -Version: 0.12.0 +Version: 0.13.0 Summary: Makes working with XML feel like you are working with JSON Home-page: https://github.com/martinblech/xmltodict Author: Martin Blech @@ -11,24 +11,25 @@ Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: MIT License Classifier: Operating System :: OS Independent Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 2 -Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.4 Classifier: Programming Language :: Python :: 3.5 Classifier: Programming Language :: Python :: 3.6 Classifier: Programming Language :: Python :: 3.7 -Classifier: Programming Language :: Python :: Implementation :: Jython +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: Implementation :: PyPy Classifier: Topic :: Text Processing :: Markup :: XML -Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* +Requires-Python: >=3.4 Description-Content-Type: text/markdown +License-File: LICENSE # xmltodict `xmltodict` is a Python module that makes working with XML feel like you are working with [JSON](http://docs.python.org/library/json.html), as in this ["spec"](http://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html): -[![Build Status](https://secure.travis-ci.org/martinblech/xmltodict.svg)](http://travis-ci.org/martinblech/xmltodict) +[![Build Status](https://travis-ci.com/martinblech/xmltodict.svg?branch=master)](https://travis-ci.com/martinblech/xmltodict) ```python >>> print(json.dumps(xmltodict.parse(""" @@ -189,6 +190,37 @@ Text values for nodes can be specified with the `cdata_key` key in the python di <text stroke="2" color="red">This is a test</text> ``` +Lists that are specified under a key in a dictionary use the key as a tag for each item. But if a list does have a parent key, for example if a list exists inside another list, it does not have a tag to use and the items are converted to a string as shown in the example below. To give tags to nested lists, use the `expand_iter` keyword argument to provide a tag as demonstrated below. Note that using `expand_iter` will break roundtripping. + +```python +>>> mydict = { +... "line": { +... "points": [ +... [1, 5], +... [2, 6], +... ] +... } +... } +>>> print(xmltodict.unparse(mydict, pretty=True)) +<?xml version="1.0" encoding="utf-8"?> +<line> + <points>[1, 5]</points> + <points>[2, 6]</points> +</line> +>>> print(xmltodict.unparse(mydict, pretty=True, expand_iter="coord")) +<?xml version="1.0" encoding="utf-8"?> +<line> + <points> + <coord>1</coord> + <coord>5</coord> + </points> + <points> + <coord>2</coord> + <coord>6</coord> + </points> +</line> +``` + ## Ok, how do I get it? ### Using pypi @@ -231,4 +263,16 @@ There is an [official FreeBSD port for xmltodict](https://svnweb.freebsd.org/por $ pkg install py36-xmltodict ``` +### openSUSE/SLE (SLE 15, Leap 15, Tumbleweed) + +There is an [official openSUSE package for xmltodict](https://software.opensuse.org/package/python-xmltodict). + +```sh +# Python2 +$ zypper in python2-xmltodict + +# Python3 +$ zypper in python3-xmltodict +``` + diff --git a/contrib/python/xmltodict/py3/README.md b/contrib/python/xmltodict/py3/README.md index 59177da90b..ab63401a80 100644 --- a/contrib/python/xmltodict/py3/README.md +++ b/contrib/python/xmltodict/py3/README.md @@ -2,7 +2,7 @@ `xmltodict` is a Python module that makes working with XML feel like you are working with [JSON](http://docs.python.org/library/json.html), as in this ["spec"](http://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html): -[![Build Status](https://secure.travis-ci.org/martinblech/xmltodict.svg)](http://travis-ci.org/martinblech/xmltodict) +[![Build Status](https://travis-ci.com/martinblech/xmltodict.svg?branch=master)](https://travis-ci.com/martinblech/xmltodict) ```python >>> print(json.dumps(xmltodict.parse(""" @@ -163,6 +163,37 @@ Text values for nodes can be specified with the `cdata_key` key in the python di <text stroke="2" color="red">This is a test</text> ``` +Lists that are specified under a key in a dictionary use the key as a tag for each item. But if a list does have a parent key, for example if a list exists inside another list, it does not have a tag to use and the items are converted to a string as shown in the example below. To give tags to nested lists, use the `expand_iter` keyword argument to provide a tag as demonstrated below. Note that using `expand_iter` will break roundtripping. + +```python +>>> mydict = { +... "line": { +... "points": [ +... [1, 5], +... [2, 6], +... ] +... } +... } +>>> print(xmltodict.unparse(mydict, pretty=True)) +<?xml version="1.0" encoding="utf-8"?> +<line> + <points>[1, 5]</points> + <points>[2, 6]</points> +</line> +>>> print(xmltodict.unparse(mydict, pretty=True, expand_iter="coord")) +<?xml version="1.0" encoding="utf-8"?> +<line> + <points> + <coord>1</coord> + <coord>5</coord> + </points> + <points> + <coord>2</coord> + <coord>6</coord> + </points> +</line> +``` + ## Ok, how do I get it? ### Using pypi @@ -204,3 +235,15 @@ There is an [official FreeBSD port for xmltodict](https://svnweb.freebsd.org/por ```sh $ pkg install py36-xmltodict ``` + +### openSUSE/SLE (SLE 15, Leap 15, Tumbleweed) + +There is an [official openSUSE package for xmltodict](https://software.opensuse.org/package/python-xmltodict). + +```sh +# Python2 +$ zypper in python2-xmltodict + +# Python3 +$ zypper in python3-xmltodict +``` diff --git a/contrib/python/xmltodict/py3/tests/test_dicttoxml.py b/contrib/python/xmltodict/py3/tests/test_dicttoxml.py index 84fa5da871..7fc21718ae 100644 --- a/contrib/python/xmltodict/py3/tests/test_dicttoxml.py +++ b/contrib/python/xmltodict/py3/tests/test_dicttoxml.py @@ -46,6 +46,14 @@ class DictToXMLTestCase(unittest.TestCase): self.assertEqual(obj, parse(unparse(obj))) self.assertEqual(unparse(obj), unparse(parse(unparse(obj)))) + def test_list_expand_iter(self): + obj = {'a': {'b': [['1', '2'], ['3',]]}} + #self.assertEqual(obj, parse(unparse(obj, expand_iter="item"))) + exp_xml = dedent('''\ + <?xml version="1.0" encoding="utf-8"?> + <a><b><item>1</item><item>2</item></b><b><item>3</item></b></a>''') + self.assertEqual(exp_xml, unparse(obj, expand_iter="item")) + def test_generator(self): obj = {'a': {'b': ['1', '2', '3']}} diff --git a/contrib/python/xmltodict/py3/tests/test_xmltodict.py b/contrib/python/xmltodict/py3/tests/test_xmltodict.py index d778816817..04137f9e03 100644 --- a/contrib/python/xmltodict/py3/tests/test_xmltodict.py +++ b/contrib/python/xmltodict/py3/tests/test_xmltodict.py @@ -1,4 +1,5 @@ from xmltodict import parse, ParsingInterrupted +import collections import unittest try: @@ -119,6 +120,17 @@ class XMLToDictTestCase(unittest.TestCase): parse, '<a>x</a>', item_depth=1, item_callback=cb) + def test_streaming_generator(self): + def cb(path, item): + cb.count += 1 + self.assertEqual(path, [('a', {'x': 'y'}), ('b', None)]) + self.assertEqual(item, str(cb.count)) + return True + cb.count = 0 + parse((n for n in '<a x="y"><b>1</b><b>2</b><b>3</b></a>'), + item_depth=2, item_callback=cb) + self.assertEqual(cb.count, 3) + def test_postprocessor(self): def postprocessor(path, key, value): try: @@ -171,7 +183,8 @@ class XMLToDictTestCase(unittest.TestCase): xml = """ <root xmlns="http://defaultns.com/" xmlns:a="http://a.com/" - xmlns:b="http://b.com/"> + xmlns:b="http://b.com/" + version="1.00"> <x a:attr="val">1</x> <a:y>2</a:y> <b:z>3</b:z> @@ -179,12 +192,13 @@ class XMLToDictTestCase(unittest.TestCase): """ d = { 'http://defaultns.com/:root': { + '@version': '1.00', + '@xmlns': { + '': 'http://defaultns.com/', + 'a': 'http://a.com/', + 'b': 'http://b.com/', + }, 'http://defaultns.com/:x': { - '@xmlns': { - '': 'http://defaultns.com/', - 'a': 'http://a.com/', - 'b': 'http://b.com/', - }, '@http://a.com/:attr': 'val', '#text': '1', }, @@ -199,7 +213,8 @@ class XMLToDictTestCase(unittest.TestCase): xml = """ <root xmlns="http://defaultns.com/" xmlns:a="http://a.com/" - xmlns:b="http://b.com/"> + xmlns:b="http://b.com/" + version="1.00"> <x a:attr="val">1</x> <a:y>2</a:y> <b:z>3</b:z> @@ -211,12 +226,13 @@ class XMLToDictTestCase(unittest.TestCase): } d = { 'root': { + '@version': '1.00', + '@xmlns': { + '': 'http://defaultns.com/', + 'a': 'http://a.com/', + 'b': 'http://b.com/', + }, 'x': { - '@xmlns': { - '': 'http://defaultns.com/', - 'a': 'http://a.com/', - 'b': 'http://b.com/', - }, '@ns_a:attr': 'val', '#text': '1', }, @@ -227,11 +243,43 @@ class XMLToDictTestCase(unittest.TestCase): res = parse(xml, process_namespaces=True, namespaces=namespaces) self.assertEqual(res, d) + def test_namespace_collapse_all(self): + xml = """ + <root xmlns="http://defaultns.com/" + xmlns:a="http://a.com/" + xmlns:b="http://b.com/" + version="1.00"> + <x a:attr="val">1</x> + <a:y>2</a:y> + <b:z>3</b:z> + </root> + """ + namespaces = collections.defaultdict(lambda: None) + d = { + 'root': { + '@version': '1.00', + '@xmlns': { + '': 'http://defaultns.com/', + 'a': 'http://a.com/', + 'b': 'http://b.com/', + }, + 'x': { + '@attr': 'val', + '#text': '1', + }, + 'y': '2', + 'z': '3', + }, + } + res = parse(xml, process_namespaces=True, namespaces=namespaces) + self.assertEqual(res, d) + def test_namespace_ignore(self): xml = """ <root xmlns="http://defaultns.com/" xmlns:a="http://a.com/" - xmlns:b="http://b.com/"> + xmlns:b="http://b.com/" + version="1.00"> <x>1</x> <a:y>2</a:y> <b:z>3</b:z> @@ -242,6 +290,7 @@ class XMLToDictTestCase(unittest.TestCase): '@xmlns': 'http://defaultns.com/', '@xmlns:a': 'http://a.com/', '@xmlns:b': 'http://b.com/', + '@version': '1.00', 'x': '1', 'a:y': '2', 'b:z': '3', @@ -380,3 +429,31 @@ class XMLToDictTestCase(unittest.TestCase): else: self.assertTrue(False) expat.ParserCreate = ParserCreate + + def test_comments(self): + xml = """ + <a> + <b> + <!-- b comment --> + <c> + <!-- c comment --> + 1 + </c> + <d>2</d> + </b> + </a> + """ + expectedResult = { + 'a': { + 'b': { + '#comment': 'b comment', + 'c': { + + '#comment': 'c comment', + '#text': '1', + }, + 'd': '2', + }, + } + } + self.assertEqual(parse(xml, process_comments=True), expectedResult) diff --git a/contrib/python/xmltodict/py3/xmltodict.py b/contrib/python/xmltodict/py3/xmltodict.py index d6dbcd7a70..ca760aa637 100644 --- a/contrib/python/xmltodict/py3/xmltodict.py +++ b/contrib/python/xmltodict/py3/xmltodict.py @@ -15,7 +15,12 @@ except ImportError: # pragma no cover except ImportError: from io import StringIO -from collections import OrderedDict +_dict = dict +import platform +if tuple(map(int, platform.python_version_tuple()[:2])) < (3, 7): + from collections import OrderedDict as _dict + +from inspect import isgenerator try: # pragma no cover _basestring = basestring @@ -27,7 +32,7 @@ except NameError: # pragma no cover _unicode = str __author__ = 'Martin Blech' -__version__ = '0.12.0' +__version__ = '0.13.0' __license__ = 'MIT' @@ -45,11 +50,12 @@ class _DictSAXHandler(object): force_cdata=False, cdata_separator='', postprocessor=None, - dict_constructor=OrderedDict, + dict_constructor=_dict, strip_whitespace=True, namespace_separator=':', namespaces=None, - force_list=None): + force_list=None, + comment_key='#comment'): self.path = [] self.stack = [] self.data = [] @@ -66,17 +72,21 @@ class _DictSAXHandler(object): self.strip_whitespace = strip_whitespace self.namespace_separator = namespace_separator self.namespaces = namespaces - self.namespace_declarations = OrderedDict() + self.namespace_declarations = dict_constructor() self.force_list = force_list + self.comment_key = comment_key def _build_name(self, full_name): - if not self.namespaces: + if self.namespaces is None: return full_name i = full_name.rfind(self.namespace_separator) if i == -1: return full_name namespace, name = full_name[:i], full_name[i+1:] - short_namespace = self.namespaces.get(namespace, namespace) + try: + short_namespace = self.namespaces[namespace] + except KeyError: + short_namespace = namespace if not short_namespace: return name else: @@ -95,7 +105,7 @@ class _DictSAXHandler(object): attrs = self._attrs_to_dict(attrs) if attrs and self.namespace_declarations: attrs['xmlns'] = self.namespace_declarations - self.namespace_declarations = OrderedDict() + self.namespace_declarations = self.dict_constructor() self.path.append((name, attrs or None)) if len(self.path) > self.item_depth: self.stack.append((self.item, self.data)) @@ -126,7 +136,7 @@ class _DictSAXHandler(object): should_continue = self.item_callback(self.path, item) if not should_continue: raise ParsingInterrupted() - if len(self.stack): + if self.stack: data = (None if not self.data else self.cdata_separator.join(self.data)) item = self.item @@ -152,6 +162,11 @@ class _DictSAXHandler(object): else: self.data.append(data) + def comments(self, data): + if self.strip_whitespace: + data = data.strip() + self.item = self.push_data(self.item, self.comment_key, data) + def push_data(self, item, key, data): if self.postprocessor is not None: result = self.postprocessor(self.path, key, data) @@ -185,10 +200,10 @@ class _DictSAXHandler(object): def parse(xml_input, encoding=None, expat=expat, process_namespaces=False, - namespace_separator=':', disable_entities=True, **kwargs): + namespace_separator=':', disable_entities=True, process_comments=False, **kwargs): """Parse the given XML input and convert it into a dictionary. - `xml_input` can either be a `string` or a file-like object. + `xml_input` can either be a `string`, a file-like object, or a generator of strings. If `xml_attribs` is `True`, element attributes are put in the dictionary among regular child elements, using `@` as a prefix to avoid collisions. If @@ -243,21 +258,21 @@ def parse(xml_input, encoding=None, expat=expat, process_namespaces=False, ... return key, value >>> xmltodict.parse('<a><b>1</b><b>2</b><b>x</b></a>', ... postprocessor=postprocessor) - OrderedDict([(u'a', OrderedDict([(u'b:int', [1, 2]), (u'b', u'x')]))]) + {'a': {'b:int': [1, 2], 'b': 'x'}} You can pass an alternate version of `expat` (such as `defusedexpat`) by using the `expat` parameter. E.g: >>> import defusedexpat >>> xmltodict.parse('<a>hello</a>', expat=defusedexpat.pyexpat) - OrderedDict([(u'a', u'hello')]) + {'a': 'hello'} You can use the force_list argument to force lists to be created even when there is only a single child of a given level of hierarchy. The force_list argument is a tuple of keys. If the key for a given level of hierarchy is in the force_list argument, that level of hierarchy will have a list as a child (even if there is only one sub-element). - The index_keys operation takes precendence over this. This is applied + The index_keys operation takes precedence over this. This is applied after any user-supplied postprocessor has already run. For example, given this input: @@ -287,6 +302,36 @@ def parse(xml_input, encoding=None, expat=expat, process_namespaces=False, `force_list` can also be a callable that receives `path`, `key` and `value`. This is helpful in cases where the logic that decides whether a list should be forced is more complex. + + + If `process_comment` is `True` then comment will be added with comment_key + (default=`'#comment'`) to then tag which contains comment + + For example, given this input: + <a> + <b> + <!-- b comment --> + <c> + <!-- c comment --> + 1 + </c> + <d>2</d> + </b> + </a> + + If called with process_comment=True, it will produce + this dictionary: + 'a': { + 'b': { + '#comment': 'b comment', + 'c': { + + '#comment': 'c comment', + '#text': '1', + }, + 'd': '2', + }, + } """ handler = _DictSAXHandler(namespace_separator=namespace_separator, **kwargs) @@ -309,6 +354,8 @@ def parse(xml_input, encoding=None, expat=expat, process_namespaces=False, parser.StartElementHandler = handler.startElement parser.EndElementHandler = handler.endElement parser.CharacterDataHandler = handler.characters + if process_comments: + parser.CommentHandler = handler.comments parser.buffer_text = True if disable_entities: try: @@ -323,6 +370,10 @@ def parse(xml_input, encoding=None, expat=expat, process_namespaces=False, parser.ExternalEntityRefHandler = lambda *x: 1 if hasattr(xml_input, 'read'): parser.ParseFile(xml_input) + elif isgenerator(xml_input): + for chunk in xml_input: + parser.Parse(chunk,False) + parser.Parse(b'',True) else: parser.Parse(xml_input, True) return handler.item @@ -353,7 +404,8 @@ def _emit(key, value, content_handler, indent='\t', namespace_separator=':', namespaces=None, - full_document=True): + full_document=True, + expand_iter=None): key = _process_namespace(key, namespaces, namespace_separator, attr_prefix) if preprocessor is not None: result = preprocessor(key, value) @@ -368,18 +420,21 @@ def _emit(key, value, content_handler, if full_document and depth == 0 and index > 0: raise ValueError('document with multiple roots') if v is None: - v = OrderedDict() + v = _dict() elif isinstance(v, bool): if v: v = _unicode('true') else: v = _unicode('false') elif not isinstance(v, dict): - v = _unicode(v) + if expand_iter and hasattr(v, '__iter__') and not isinstance(v, _basestring): + v = _dict(((expand_iter, v),)) + else: + v = _unicode(v) if isinstance(v, _basestring): - v = OrderedDict(((cdata_key, v),)) + v = _dict(((cdata_key, v),)) cdata = None - attrs = OrderedDict() + attrs = _dict() children = [] for ik, iv in v.items(): if ik == cdata_key: @@ -407,7 +462,8 @@ def _emit(key, value, content_handler, _emit(child_key, child_value, content_handler, attr_prefix, cdata_key, depth+1, preprocessor, pretty, newl, indent, namespaces=namespaces, - namespace_separator=namespace_separator) + namespace_separator=namespace_separator, + expand_iter=expand_iter) if cdata is not None: content_handler.characters(cdata) if pretty and children: diff --git a/ydb/tests/functional/sqs/test_generic_messaging.py b/ydb/tests/functional/sqs/test_generic_messaging.py index 3f6b2371ba..1b1838ce58 100644 --- a/ydb/tests/functional/sqs/test_generic_messaging.py +++ b/ydb/tests/functional/sqs/test_generic_messaging.py @@ -3,7 +3,6 @@ import base64 import logging import time -from collections import OrderedDict import pytest from hamcrest import assert_that, equal_to, not_none, greater_than, has_item, has_items, raises, is_not, not_, empty, instance_of @@ -944,7 +943,7 @@ class SqsGenericMessagingTest(KikimrSqsTestBase): len(batch_result['GetQueueAttributesBatchResultEntry']), equal_to(11) ) assert_that( - batch_result['BatchResultErrorEntry'], instance_of(OrderedDict) # that means that we have only one entry for error + batch_result['BatchResultErrorEntry'], instance_of(dict) # that means that we have only one entry for error ) for entry in batch_result['GetQueueAttributesBatchResultEntry']: diff --git a/ydb/tests/functional/sqs/test_queues_managing.py b/ydb/tests/functional/sqs/test_queues_managing.py index bcfd91ca67..62bc631f02 100644 --- a/ydb/tests/functional/sqs/test_queues_managing.py +++ b/ydb/tests/functional/sqs/test_queues_managing.py @@ -1,7 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- import time -from collections import OrderedDict import pytest from hamcrest import assert_that, equal_to, greater_than, not_none, none, has_item, has_items, raises, empty, instance_of @@ -163,7 +162,7 @@ class QueuesManagingTest(KikimrSqsTestBase): len(delete_queue_batch_result['DeleteQueueBatchResultEntry']), equal_to(2) ) assert_that( - delete_queue_batch_result['BatchResultErrorEntry'], instance_of(OrderedDict) # that means that we have only one entry for error + delete_queue_batch_result['BatchResultErrorEntry'], instance_of(dict) # that means that we have only one entry for error ) existing_queues = [to_bytes(y) for y in self._sqs_api.list_queues()] @@ -217,7 +216,7 @@ class QueuesManagingTest(KikimrSqsTestBase): len(purge_queue_batch_result['PurgeQueueBatchResultEntry']), equal_to(2) ) assert_that( - purge_queue_batch_result['BatchResultErrorEntry'], instance_of(OrderedDict) # that means that we have only one entry for error + purge_queue_batch_result['BatchResultErrorEntry'], instance_of(dict) # that means that we have only one entry for error ) def check_purged_queue(queue_url): |