From bb139d04e93ed3e06df4dcc138e8cd2e8e986885 Mon Sep 17 00:00:00 2001 From: arcadia-devtools Date: Wed, 11 May 2022 11:11:25 +0300 Subject: intermediate changes ref:89be2b65e3d35e908aa2d4047a4396bbda1ab868 --- contrib/python/xmltodict/.dist-info/METADATA | 234 ---------- contrib/python/xmltodict/.dist-info/top_level.txt | 1 - contrib/python/xmltodict/LICENSE | 7 - contrib/python/xmltodict/README.md | 206 --------- contrib/python/xmltodict/patches/01-pr-290.patch | 162 ------- contrib/python/xmltodict/py2/.dist-info/METADATA | 234 ++++++++++ .../python/xmltodict/py2/.dist-info/top_level.txt | 1 + contrib/python/xmltodict/py2/LICENSE | 7 + contrib/python/xmltodict/py2/README.md | 206 +++++++++ .../python/xmltodict/py2/tests/test_dicttoxml.py | 207 +++++++++ .../python/xmltodict/py2/tests/test_xmltodict.py | 382 ++++++++++++++++ contrib/python/xmltodict/py2/xmltodict.py | 488 +++++++++++++++++++++ contrib/python/xmltodict/py3/.dist-info/METADATA | 234 ++++++++++ .../python/xmltodict/py3/.dist-info/top_level.txt | 1 + contrib/python/xmltodict/py3/LICENSE | 7 + contrib/python/xmltodict/py3/README.md | 206 +++++++++ .../python/xmltodict/py3/tests/test_dicttoxml.py | 207 +++++++++ .../python/xmltodict/py3/tests/test_xmltodict.py | 382 ++++++++++++++++ contrib/python/xmltodict/py3/xmltodict.py | 488 +++++++++++++++++++++ contrib/python/xmltodict/tests/test_dicttoxml.py | 207 --------- contrib/python/xmltodict/tests/test_xmltodict.py | 382 ---------------- contrib/python/xmltodict/xmltodict.py | 488 --------------------- 22 files changed, 3050 insertions(+), 1687 deletions(-) delete mode 100644 contrib/python/xmltodict/.dist-info/METADATA delete mode 100644 contrib/python/xmltodict/.dist-info/top_level.txt delete mode 100644 contrib/python/xmltodict/LICENSE delete mode 100644 contrib/python/xmltodict/README.md delete mode 100644 contrib/python/xmltodict/patches/01-pr-290.patch create mode 100644 contrib/python/xmltodict/py2/.dist-info/METADATA create mode 100644 contrib/python/xmltodict/py2/.dist-info/top_level.txt create mode 100644 contrib/python/xmltodict/py2/LICENSE create mode 100644 contrib/python/xmltodict/py2/README.md create mode 100644 contrib/python/xmltodict/py2/tests/test_dicttoxml.py create mode 100644 contrib/python/xmltodict/py2/tests/test_xmltodict.py create mode 100644 contrib/python/xmltodict/py2/xmltodict.py create mode 100644 contrib/python/xmltodict/py3/.dist-info/METADATA create mode 100644 contrib/python/xmltodict/py3/.dist-info/top_level.txt create mode 100644 contrib/python/xmltodict/py3/LICENSE create mode 100644 contrib/python/xmltodict/py3/README.md create mode 100644 contrib/python/xmltodict/py3/tests/test_dicttoxml.py create mode 100644 contrib/python/xmltodict/py3/tests/test_xmltodict.py create mode 100644 contrib/python/xmltodict/py3/xmltodict.py delete mode 100644 contrib/python/xmltodict/tests/test_dicttoxml.py delete mode 100644 contrib/python/xmltodict/tests/test_xmltodict.py delete mode 100644 contrib/python/xmltodict/xmltodict.py (limited to 'contrib/python') diff --git a/contrib/python/xmltodict/.dist-info/METADATA b/contrib/python/xmltodict/.dist-info/METADATA deleted file mode 100644 index 73ad4cc22f8..00000000000 --- a/contrib/python/xmltodict/.dist-info/METADATA +++ /dev/null @@ -1,234 +0,0 @@ -Metadata-Version: 2.1 -Name: xmltodict -Version: 0.12.0 -Summary: Makes working with XML feel like you are working with JSON -Home-page: https://github.com/martinblech/xmltodict -Author: Martin Blech -Author-email: martinblech@gmail.com -License: MIT -Platform: all -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: MIT License -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 2 -Classifier: Programming Language :: Python :: 2.7 -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.4 -Classifier: Programming Language :: Python :: 3.5 -Classifier: Programming Language :: Python :: 3.6 -Classifier: Programming Language :: Python :: 3.7 -Classifier: Programming Language :: Python :: Implementation :: Jython -Classifier: Programming Language :: Python :: Implementation :: PyPy -Classifier: Topic :: Text Processing :: Markup :: XML -Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* -Description-Content-Type: text/markdown - -# xmltodict - -`xmltodict` is a Python module that makes working with XML feel like you are working with [JSON](http://docs.python.org/library/json.html), as in this ["spec"](http://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html): - -[![Build Status](https://secure.travis-ci.org/martinblech/xmltodict.svg)](http://travis-ci.org/martinblech/xmltodict) - -```python ->>> print(json.dumps(xmltodict.parse(""" -... -... -... elements -... more elements -... -... -... element as well -... -... -... """), indent=4)) -{ - "mydocument": { - "@has": "an attribute", - "and": { - "many": [ - "elements", - "more elements" - ] - }, - "plus": { - "@a": "complex", - "#text": "element as well" - } - } -} -``` - -## Namespace support - -By default, `xmltodict` does no XML namespace processing (it just treats namespace declarations as regular node attributes), but passing `process_namespaces=True` will make it expand namespaces for you: - -```python ->>> xml = """ -... -... 1 -... 2 -... 3 -... -... """ ->>> xmltodict.parse(xml, process_namespaces=True) == { -... 'http://defaultns.com/:root': { -... 'http://defaultns.com/:x': '1', -... 'http://a.com/:y': '2', -... 'http://b.com/:z': '3', -... } -... } -True -``` - -It also lets you collapse certain namespaces to shorthand prefixes, or skip them altogether: - -```python ->>> namespaces = { -... 'http://defaultns.com/': None, # skip this namespace -... 'http://a.com/': 'ns_a', # collapse "http://a.com/" -> "ns_a" -... } ->>> xmltodict.parse(xml, process_namespaces=True, namespaces=namespaces) == { -... 'root': { -... 'x': '1', -... 'ns_a:y': '2', -... 'http://b.com/:z': '3', -... }, -... } -True -``` - -## Streaming mode - -`xmltodict` is very fast ([Expat](http://docs.python.org/library/pyexpat.html)-based) and has a streaming mode with a small memory footprint, suitable for big XML dumps like [Discogs](http://discogs.com/data/) or [Wikipedia](http://dumps.wikimedia.org/): - -```python ->>> def handle_artist(_, artist): -... print(artist['name']) -... return True ->>> ->>> xmltodict.parse(GzipFile('discogs_artists.xml.gz'), -... item_depth=2, item_callback=handle_artist) -A Perfect Circle -Fantômas -King Crimson -Chris Potter -... -``` - -It can also be used from the command line to pipe objects to a script like this: - -```python -import sys, marshal -while True: - _, article = marshal.load(sys.stdin) - print(article['title']) -``` - -```sh -$ bunzip2 enwiki-pages-articles.xml.bz2 | xmltodict.py 2 | myscript.py -AccessibleComputing -Anarchism -AfghanistanHistory -AfghanistanGeography -AfghanistanPeople -AfghanistanCommunications -Autism -... -``` - -Or just cache the dicts so you don't have to parse that big XML file again. You do this only once: - -```sh -$ bunzip2 enwiki-pages-articles.xml.bz2 | xmltodict.py 2 | gzip > enwiki.dicts.gz -``` - -And you reuse the dicts with every script that needs them: - -```sh -$ gunzip enwiki.dicts.gz | script1.py -$ gunzip enwiki.dicts.gz | script2.py -... -``` - -## Roundtripping - -You can also convert in the other direction, using the `unparse()` method: - -```python ->>> mydict = { -... 'response': { -... 'status': 'good', -... 'last_updated': '2014-02-16T23:10:12Z', -... } -... } ->>> print(unparse(mydict, pretty=True)) - - - good - 2014-02-16T23:10:12Z - -``` - -Text values for nodes can be specified with the `cdata_key` key in the python dict, while node properties can be specified with the `attr_prefix` prefixed to the key name in the python dict. The default value for `attr_prefix` is `@` and the default value for `cdata_key` is `#text`. - -```python ->>> import xmltodict ->>> ->>> mydict = { -... 'text': { -... '@color':'red', -... '@stroke':'2', -... '#text':'This is a test' -... } -... } ->>> print(xmltodict.unparse(mydict, pretty=True)) - -This is a test -``` - -## Ok, how do I get it? - -### Using pypi - -You just need to - -```sh -$ pip install xmltodict -``` - -### RPM-based distro (Fedora, RHEL, …) - -There is an [official Fedora package for xmltodict](https://apps.fedoraproject.org/packages/python-xmltodict). - -```sh -$ sudo yum install python-xmltodict -``` - -### Arch Linux - -There is an [official Arch Linux package for xmltodict](https://www.archlinux.org/packages/community/any/python-xmltodict/). - -```sh -$ sudo pacman -S python-xmltodict -``` - -### Debian-based distro (Debian, Ubuntu, …) - -There is an [official Debian package for xmltodict](https://tracker.debian.org/pkg/python-xmltodict). - -```sh -$ sudo apt install python-xmltodict -``` - -### FreeBSD - -There is an [official FreeBSD port for xmltodict](https://svnweb.freebsd.org/ports/head/devel/py-xmltodict/). - -```sh -$ pkg install py36-xmltodict -``` - - diff --git a/contrib/python/xmltodict/.dist-info/top_level.txt b/contrib/python/xmltodict/.dist-info/top_level.txt deleted file mode 100644 index a0d4bb760b6..00000000000 --- a/contrib/python/xmltodict/.dist-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -xmltodict diff --git a/contrib/python/xmltodict/LICENSE b/contrib/python/xmltodict/LICENSE deleted file mode 100644 index a462778cff0..00000000000 --- a/contrib/python/xmltodict/LICENSE +++ /dev/null @@ -1,7 +0,0 @@ -Copyright (C) 2012 Martin Blech and individual contributors. - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/contrib/python/xmltodict/README.md b/contrib/python/xmltodict/README.md deleted file mode 100644 index ba3397e556e..00000000000 --- a/contrib/python/xmltodict/README.md +++ /dev/null @@ -1,206 +0,0 @@ -# xmltodict - -`xmltodict` is a Python module that makes working with XML feel like you are working with [JSON](http://docs.python.org/library/json.html), as in this ["spec"](http://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html): - -[![Build Status](https://secure.travis-ci.org/martinblech/xmltodict.svg)](http://travis-ci.org/martinblech/xmltodict) - -```python ->>> print(json.dumps(xmltodict.parse(""" -... -... -... elements -... more elements -... -... -... element as well -... -... -... """), indent=4)) -{ - "mydocument": { - "@has": "an attribute", - "and": { - "many": [ - "elements", - "more elements" - ] - }, - "plus": { - "@a": "complex", - "#text": "element as well" - } - } -} -``` - -## Namespace support - -By default, `xmltodict` does no XML namespace processing (it just treats namespace declarations as regular node attributes), but passing `process_namespaces=True` will make it expand namespaces for you: - -```python ->>> xml = """ -... -... 1 -... 2 -... 3 -... -... """ ->>> xmltodict.parse(xml, process_namespaces=True) == { -... 'http://defaultns.com/|root': { -... 'http://defaultns.com/|x': '1', -... 'http://a.com/|y': '2', -... 'http://b.com/|z': '3', -... } -... } -True -``` - -It also lets you collapse certain namespaces to shorthand prefixes, or skip them altogether: - -```python ->>> namespaces = { -... 'http://defaultns.com/': None, # skip this namespace -... 'http://a.com/': 'ns_a', # collapse "http://a.com/" -> "ns_a" -... } ->>> xmltodict.parse(xml, process_namespaces=True, namespaces=namespaces) == { -... 'root': { -... 'x': '1', -... 'ns_a|y': '2', -... 'http://b.com/|z': '3', -... }, -... } -True -``` - -## Streaming mode - -`xmltodict` is very fast ([Expat](http://docs.python.org/library/pyexpat.html)-based) and has a streaming mode with a small memory footprint, suitable for big XML dumps like [Discogs](http://discogs.com/data/) or [Wikipedia](http://dumps.wikimedia.org/): - -```python ->>> def handle_artist(_, artist): -... print(artist['name']) -... return True ->>> ->>> xmltodict.parse(GzipFile('discogs_artists.xml.gz'), -... item_depth=2, item_callback=handle_artist) -A Perfect Circle -Fantômas -King Crimson -Chris Potter -... -``` - -It can also be used from the command line to pipe objects to a script like this: - -```python -import sys, marshal -while True: - _, article = marshal.load(sys.stdin) - print(article['title']) -``` - -```sh -$ bunzip2 enwiki-pages-articles.xml.bz2 | xmltodict.py 2 | myscript.py -AccessibleComputing -Anarchism -AfghanistanHistory -AfghanistanGeography -AfghanistanPeople -AfghanistanCommunications -Autism -... -``` - -Or just cache the dicts so you don't have to parse that big XML file again. You do this only once: - -```sh -$ bunzip2 enwiki-pages-articles.xml.bz2 | xmltodict.py 2 | gzip > enwiki.dicts.gz -``` - -And you reuse the dicts with every script that needs them: - -```sh -$ gunzip enwiki.dicts.gz | script1.py -$ gunzip enwiki.dicts.gz | script2.py -... -``` - -## Roundtripping - -You can also convert in the other direction, using the `unparse()` method: - -```python ->>> mydict = { -... 'response': { -... 'status': 'good', -... 'last_updated': '2014-02-16T23:10:12Z', -... } -... } ->>> print(unparse(mydict, pretty=True)) - - - good - 2014-02-16T23:10:12Z - -``` - -Text values for nodes can be specified with the `cdata_key` key in the python dict, while node properties can be specified with the `attr_prefix` prefixed to the key name in the python dict. The default value for `attr_prefix` is `@` and the default value for `cdata_key` is `#text`. - -```python ->>> import xmltodict ->>> ->>> mydict = { -... 'text': { -... '@color':'red', -... '@stroke':'2', -... '#text':'This is a test' -... } -... } ->>> print(xmltodict.unparse(mydict, pretty=True)) - -This is a test -``` - -## Ok, how do I get it? - -### Using pypi - -You just need to - -```sh -$ pip install xmltodict -``` - -### RPM-based distro (Fedora, RHEL, …) - -There is an [official Fedora package for xmltodict](https://apps.fedoraproject.org/packages/python-xmltodict). - -```sh -$ sudo yum install python-xmltodict -``` - -### Arch Linux - -There is an [official Arch Linux package for xmltodict](https://www.archlinux.org/packages/community/any/python-xmltodict/). - -```sh -$ sudo pacman -S python-xmltodict -``` - -### Debian-based distro (Debian, Ubuntu, …) - -There is an [official Debian package for xmltodict](https://tracker.debian.org/pkg/python-xmltodict). - -```sh -$ sudo apt install python-xmltodict -``` - -### FreeBSD - -There is an [official FreeBSD port for xmltodict](https://svnweb.freebsd.org/ports/head/devel/py-xmltodict/). - -```sh -$ pkg install py36-xmltodict -``` diff --git a/contrib/python/xmltodict/patches/01-pr-290.patch b/contrib/python/xmltodict/patches/01-pr-290.patch deleted file mode 100644 index de530c96edd..00000000000 --- a/contrib/python/xmltodict/patches/01-pr-290.patch +++ /dev/null @@ -1,162 +0,0 @@ -From b9f4e5f91e1821dbffe3db27c1dd6b6e76f4cbec Mon Sep 17 00:00:00 2001 -From: Sebastian Pipping -Date: Sun, 27 Feb 2022 02:07:55 +0100 -Subject: [PATCH] Switch namespace separator from colon to pipe - ---- - README.md | 12 ++++++------ - tests/test_dicttoxml.py | 10 +++++----- - tests/test_xmltodict.py | 16 ++++++++-------- - xmltodict.py | 10 +++++----- - 4 files changed, 24 insertions(+), 24 deletions(-) - -diff --git a/README.md b/README.md -index ab63401..9309837 100644 ---- contrib/python/xmltodict/README.md -+++ contrib/python/xmltodict/README.md -@@ -48,10 +48,10 @@ By default, `xmltodict` does no XML namespace processing (it just treats namespa - ... - ... """ - >>> xmltodict.parse(xml, process_namespaces=True) == { --... 'http://defaultns.com/:root': { --... 'http://defaultns.com/:x': '1', --... 'http://a.com/:y': '2', --... 'http://b.com/:z': '3', -+... 'http://defaultns.com/|root': { -+... 'http://defaultns.com/|x': '1', -+... 'http://a.com/|y': '2', -+... 'http://b.com/|z': '3', - ... } - ... } - True -@@ -67,8 +67,8 @@ It also lets you collapse certain namespaces to shorthand prefixes, or skip them - >>> xmltodict.parse(xml, process_namespaces=True, namespaces=namespaces) == { - ... 'root': { - ... 'x': '1', --... 'ns_a:y': '2', --... 'http://b.com/:z': '3', -+... 'ns_a|y': '2', -+... 'http://b.com/|z': '3', - ... }, - ... } - True -diff --git a/tests/test_dicttoxml.py b/tests/test_dicttoxml.py -index 7fc2171..dd74989 100644 ---- contrib/python/xmltodict/tests/test_dicttoxml.py -+++ contrib/python/xmltodict/tests/test_dicttoxml.py -@@ -178,18 +178,18 @@ def test_short_empty_elements(self): - - def test_namespace_support(self): - obj = OrderedDict(( -- ('http://defaultns.com/:root', OrderedDict(( -+ ('http://defaultns.com/|root', OrderedDict(( - ('@xmlns', OrderedDict(( - ('', 'http://defaultns.com/'), - ('a', 'http://a.com/'), - ('b', 'http://b.com/'), - ))), -- ('http://defaultns.com/:x', OrderedDict(( -- ('@http://a.com/:attr', 'val'), -+ ('http://defaultns.com/|x', OrderedDict(( -+ ('@http://a.com/|attr', 'val'), - ('#text', '1'), - ))), -- ('http://a.com/:y', '2'), -- ('http://b.com/:z', '3'), -+ ('http://a.com/|y', '2'), -+ ('http://b.com/|z', '3'), - ))), - )) - ns = { -diff --git a/tests/test_xmltodict.py b/tests/test_xmltodict.py -index 0455d96..762768a 100644 ---- contrib/python/xmltodict/tests/test_xmltodict.py -+++ contrib/python/xmltodict/tests/test_xmltodict.py -@@ -190,18 +190,18 @@ def test_namespace_support(self): - - """ - d = { -- 'http://defaultns.com/:root': { -- 'http://defaultns.com/:x': { -+ 'http://defaultns.com/|root': { -+ 'http://defaultns.com/|x': { - '@xmlns': { - '': 'http://defaultns.com/', - 'a': 'http://a.com/', - 'b': 'http://b.com/', - }, -- '@http://a.com/:attr': 'val', -+ '@http://a.com/|attr': 'val', - '#text': '1', - }, -- 'http://a.com/:y': '2', -- 'http://b.com/:z': '3', -+ 'http://a.com/|y': '2', -+ 'http://b.com/|z': '3', - } - } - res = parse(xml, process_namespaces=True) -@@ -229,11 +229,11 @@ def test_namespace_collapse(self): - 'a': 'http://a.com/', - 'b': 'http://b.com/', - }, -- '@ns_a:attr': 'val', -+ '@ns_a|attr': 'val', - '#text': '1', - }, -- 'ns_a:y': '2', -- 'http://b.com/:z': '3', -+ 'ns_a|y': '2', -+ 'http://b.com/|z': '3', - }, - } - res = parse(xml, process_namespaces=True, namespaces=namespaces) -diff --git a/xmltodict.py b/xmltodict.py -index a070961..b7577e1 100755 ---- contrib/python/xmltodict/xmltodict.py -+++ contrib/python/xmltodict/xmltodict.py -@@ -48,7 +48,7 @@ def __init__(self, - postprocessor=None, - dict_constructor=OrderedDict, - strip_whitespace=True, -- namespace_separator=':', -+ namespace_separator='|', - namespaces=None, - force_list=None, - comment_key='#comment'): -@@ -196,7 +196,7 @@ def _should_force_list(self, key, value): - - - def parse(xml_input, encoding=None, expat=expat, process_namespaces=False, -- namespace_separator=':', disable_entities=True, **kwargs): -+ namespace_separator='|', disable_entities=True, **kwargs): - """Parse the given XML input and convert it into a dictionary. - - `xml_input` can either be a `string`, a file-like object, or a generator of strings. -@@ -375,7 +375,7 @@ def parse(xml_input, encoding=None, expat=expat, process_namespaces=False, - return handler.item - - --def _process_namespace(name, namespaces, ns_sep=':', attr_prefix='@'): -+def _process_namespace(name, namespaces, ns_sep='|', attr_prefix='@'): - if not namespaces: - return name - try: -@@ -386,7 +386,7 @@ def _process_namespace(name, namespaces, ns_sep=':', attr_prefix='@'): - ns_res = namespaces.get(ns.strip(attr_prefix)) - name = '{}{}{}{}'.format( - attr_prefix if ns.startswith(attr_prefix) else '', -- ns_res, ns_sep, name) if ns_res else name -+ ns_res, ':', name) if ns_res else name - return name - - -@@ -398,7 +398,7 @@ def _emit(key, value, content_handler, - pretty=False, - newl='\n', - indent='\t', -- namespace_separator=':', -+ namespace_separator='|', - namespaces=None, - full_document=True, - expand_iter=None): diff --git a/contrib/python/xmltodict/py2/.dist-info/METADATA b/contrib/python/xmltodict/py2/.dist-info/METADATA new file mode 100644 index 00000000000..73ad4cc22f8 --- /dev/null +++ b/contrib/python/xmltodict/py2/.dist-info/METADATA @@ -0,0 +1,234 @@ +Metadata-Version: 2.1 +Name: xmltodict +Version: 0.12.0 +Summary: Makes working with XML feel like you are working with JSON +Home-page: https://github.com/martinblech/xmltodict +Author: Martin Blech +Author-email: martinblech@gmail.com +License: MIT +Platform: all +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: Implementation :: Jython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Text Processing :: Markup :: XML +Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* +Description-Content-Type: text/markdown + +# xmltodict + +`xmltodict` is a Python module that makes working with XML feel like you are working with [JSON](http://docs.python.org/library/json.html), as in this ["spec"](http://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html): + +[![Build Status](https://secure.travis-ci.org/martinblech/xmltodict.svg)](http://travis-ci.org/martinblech/xmltodict) + +```python +>>> print(json.dumps(xmltodict.parse(""" +... +... +... elements +... more elements +... +... +... element as well +... +... +... """), indent=4)) +{ + "mydocument": { + "@has": "an attribute", + "and": { + "many": [ + "elements", + "more elements" + ] + }, + "plus": { + "@a": "complex", + "#text": "element as well" + } + } +} +``` + +## Namespace support + +By default, `xmltodict` does no XML namespace processing (it just treats namespace declarations as regular node attributes), but passing `process_namespaces=True` will make it expand namespaces for you: + +```python +>>> xml = """ +... +... 1 +... 2 +... 3 +... +... """ +>>> xmltodict.parse(xml, process_namespaces=True) == { +... 'http://defaultns.com/:root': { +... 'http://defaultns.com/:x': '1', +... 'http://a.com/:y': '2', +... 'http://b.com/:z': '3', +... } +... } +True +``` + +It also lets you collapse certain namespaces to shorthand prefixes, or skip them altogether: + +```python +>>> namespaces = { +... 'http://defaultns.com/': None, # skip this namespace +... 'http://a.com/': 'ns_a', # collapse "http://a.com/" -> "ns_a" +... } +>>> xmltodict.parse(xml, process_namespaces=True, namespaces=namespaces) == { +... 'root': { +... 'x': '1', +... 'ns_a:y': '2', +... 'http://b.com/:z': '3', +... }, +... } +True +``` + +## Streaming mode + +`xmltodict` is very fast ([Expat](http://docs.python.org/library/pyexpat.html)-based) and has a streaming mode with a small memory footprint, suitable for big XML dumps like [Discogs](http://discogs.com/data/) or [Wikipedia](http://dumps.wikimedia.org/): + +```python +>>> def handle_artist(_, artist): +... print(artist['name']) +... return True +>>> +>>> xmltodict.parse(GzipFile('discogs_artists.xml.gz'), +... item_depth=2, item_callback=handle_artist) +A Perfect Circle +Fantômas +King Crimson +Chris Potter +... +``` + +It can also be used from the command line to pipe objects to a script like this: + +```python +import sys, marshal +while True: + _, article = marshal.load(sys.stdin) + print(article['title']) +``` + +```sh +$ bunzip2 enwiki-pages-articles.xml.bz2 | xmltodict.py 2 | myscript.py +AccessibleComputing +Anarchism +AfghanistanHistory +AfghanistanGeography +AfghanistanPeople +AfghanistanCommunications +Autism +... +``` + +Or just cache the dicts so you don't have to parse that big XML file again. You do this only once: + +```sh +$ bunzip2 enwiki-pages-articles.xml.bz2 | xmltodict.py 2 | gzip > enwiki.dicts.gz +``` + +And you reuse the dicts with every script that needs them: + +```sh +$ gunzip enwiki.dicts.gz | script1.py +$ gunzip enwiki.dicts.gz | script2.py +... +``` + +## Roundtripping + +You can also convert in the other direction, using the `unparse()` method: + +```python +>>> mydict = { +... 'response': { +... 'status': 'good', +... 'last_updated': '2014-02-16T23:10:12Z', +... } +... } +>>> print(unparse(mydict, pretty=True)) + + + good + 2014-02-16T23:10:12Z + +``` + +Text values for nodes can be specified with the `cdata_key` key in the python dict, while node properties can be specified with the `attr_prefix` prefixed to the key name in the python dict. The default value for `attr_prefix` is `@` and the default value for `cdata_key` is `#text`. + +```python +>>> import xmltodict +>>> +>>> mydict = { +... 'text': { +... '@color':'red', +... '@stroke':'2', +... '#text':'This is a test' +... } +... } +>>> print(xmltodict.unparse(mydict, pretty=True)) + +This is a test +``` + +## Ok, how do I get it? + +### Using pypi + +You just need to + +```sh +$ pip install xmltodict +``` + +### RPM-based distro (Fedora, RHEL, …) + +There is an [official Fedora package for xmltodict](https://apps.fedoraproject.org/packages/python-xmltodict). + +```sh +$ sudo yum install python-xmltodict +``` + +### Arch Linux + +There is an [official Arch Linux package for xmltodict](https://www.archlinux.org/packages/community/any/python-xmltodict/). + +```sh +$ sudo pacman -S python-xmltodict +``` + +### Debian-based distro (Debian, Ubuntu, …) + +There is an [official Debian package for xmltodict](https://tracker.debian.org/pkg/python-xmltodict). + +```sh +$ sudo apt install python-xmltodict +``` + +### FreeBSD + +There is an [official FreeBSD port for xmltodict](https://svnweb.freebsd.org/ports/head/devel/py-xmltodict/). + +```sh +$ pkg install py36-xmltodict +``` + + diff --git a/contrib/python/xmltodict/py2/.dist-info/top_level.txt b/contrib/python/xmltodict/py2/.dist-info/top_level.txt new file mode 100644 index 00000000000..a0d4bb760b6 --- /dev/null +++ b/contrib/python/xmltodict/py2/.dist-info/top_level.txt @@ -0,0 +1 @@ +xmltodict diff --git a/contrib/python/xmltodict/py2/LICENSE b/contrib/python/xmltodict/py2/LICENSE new file mode 100644 index 00000000000..a462778cff0 --- /dev/null +++ b/contrib/python/xmltodict/py2/LICENSE @@ -0,0 +1,7 @@ +Copyright (C) 2012 Martin Blech and individual contributors. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/contrib/python/xmltodict/py2/README.md b/contrib/python/xmltodict/py2/README.md new file mode 100644 index 00000000000..59177da90b7 --- /dev/null +++ b/contrib/python/xmltodict/py2/README.md @@ -0,0 +1,206 @@ +# xmltodict + +`xmltodict` is a Python module that makes working with XML feel like you are working with [JSON](http://docs.python.org/library/json.html), as in this ["spec"](http://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html): + +[![Build Status](https://secure.travis-ci.org/martinblech/xmltodict.svg)](http://travis-ci.org/martinblech/xmltodict) + +```python +>>> print(json.dumps(xmltodict.parse(""" +... +... +... elements +... more elements +... +... +... element as well +... +... +... """), indent=4)) +{ + "mydocument": { + "@has": "an attribute", + "and": { + "many": [ + "elements", + "more elements" + ] + }, + "plus": { + "@a": "complex", + "#text": "element as well" + } + } +} +``` + +## Namespace support + +By default, `xmltodict` does no XML namespace processing (it just treats namespace declarations as regular node attributes), but passing `process_namespaces=True` will make it expand namespaces for you: + +```python +>>> xml = """ +... +... 1 +... 2 +... 3 +... +... """ +>>> xmltodict.parse(xml, process_namespaces=True) == { +... 'http://defaultns.com/:root': { +... 'http://defaultns.com/:x': '1', +... 'http://a.com/:y': '2', +... 'http://b.com/:z': '3', +... } +... } +True +``` + +It also lets you collapse certain namespaces to shorthand prefixes, or skip them altogether: + +```python +>>> namespaces = { +... 'http://defaultns.com/': None, # skip this namespace +... 'http://a.com/': 'ns_a', # collapse "http://a.com/" -> "ns_a" +... } +>>> xmltodict.parse(xml, process_namespaces=True, namespaces=namespaces) == { +... 'root': { +... 'x': '1', +... 'ns_a:y': '2', +... 'http://b.com/:z': '3', +... }, +... } +True +``` + +## Streaming mode + +`xmltodict` is very fast ([Expat](http://docs.python.org/library/pyexpat.html)-based) and has a streaming mode with a small memory footprint, suitable for big XML dumps like [Discogs](http://discogs.com/data/) or [Wikipedia](http://dumps.wikimedia.org/): + +```python +>>> def handle_artist(_, artist): +... print(artist['name']) +... return True +>>> +>>> xmltodict.parse(GzipFile('discogs_artists.xml.gz'), +... item_depth=2, item_callback=handle_artist) +A Perfect Circle +Fantômas +King Crimson +Chris Potter +... +``` + +It can also be used from the command line to pipe objects to a script like this: + +```python +import sys, marshal +while True: + _, article = marshal.load(sys.stdin) + print(article['title']) +``` + +```sh +$ bunzip2 enwiki-pages-articles.xml.bz2 | xmltodict.py 2 | myscript.py +AccessibleComputing +Anarchism +AfghanistanHistory +AfghanistanGeography +AfghanistanPeople +AfghanistanCommunications +Autism +... +``` + +Or just cache the dicts so you don't have to parse that big XML file again. You do this only once: + +```sh +$ bunzip2 enwiki-pages-articles.xml.bz2 | xmltodict.py 2 | gzip > enwiki.dicts.gz +``` + +And you reuse the dicts with every script that needs them: + +```sh +$ gunzip enwiki.dicts.gz | script1.py +$ gunzip enwiki.dicts.gz | script2.py +... +``` + +## Roundtripping + +You can also convert in the other direction, using the `unparse()` method: + +```python +>>> mydict = { +... 'response': { +... 'status': 'good', +... 'last_updated': '2014-02-16T23:10:12Z', +... } +... } +>>> print(unparse(mydict, pretty=True)) + + + good + 2014-02-16T23:10:12Z + +``` + +Text values for nodes can be specified with the `cdata_key` key in the python dict, while node properties can be specified with the `attr_prefix` prefixed to the key name in the python dict. The default value for `attr_prefix` is `@` and the default value for `cdata_key` is `#text`. + +```python +>>> import xmltodict +>>> +>>> mydict = { +... 'text': { +... '@color':'red', +... '@stroke':'2', +... '#text':'This is a test' +... } +... } +>>> print(xmltodict.unparse(mydict, pretty=True)) + +This is a test +``` + +## Ok, how do I get it? + +### Using pypi + +You just need to + +```sh +$ pip install xmltodict +``` + +### RPM-based distro (Fedora, RHEL, …) + +There is an [official Fedora package for xmltodict](https://apps.fedoraproject.org/packages/python-xmltodict). + +```sh +$ sudo yum install python-xmltodict +``` + +### Arch Linux + +There is an [official Arch Linux package for xmltodict](https://www.archlinux.org/packages/community/any/python-xmltodict/). + +```sh +$ sudo pacman -S python-xmltodict +``` + +### Debian-based distro (Debian, Ubuntu, …) + +There is an [official Debian package for xmltodict](https://tracker.debian.org/pkg/python-xmltodict). + +```sh +$ sudo apt install python-xmltodict +``` + +### FreeBSD + +There is an [official FreeBSD port for xmltodict](https://svnweb.freebsd.org/ports/head/devel/py-xmltodict/). + +```sh +$ pkg install py36-xmltodict +``` diff --git a/contrib/python/xmltodict/py2/tests/test_dicttoxml.py b/contrib/python/xmltodict/py2/tests/test_dicttoxml.py new file mode 100644 index 00000000000..84fa5da871b --- /dev/null +++ b/contrib/python/xmltodict/py2/tests/test_dicttoxml.py @@ -0,0 +1,207 @@ +import sys +from xmltodict import parse, unparse +from collections import OrderedDict + +import unittest +import re +from textwrap import dedent + +IS_JYTHON = sys.platform.startswith('java') + +_HEADER_RE = re.compile(r'^[^\n]*\n') + + +def _strip(fullxml): + return _HEADER_RE.sub('', fullxml) + + +class DictToXMLTestCase(unittest.TestCase): + def test_root(self): + obj = {'a': None} + self.assertEqual(obj, parse(unparse(obj))) + self.assertEqual(unparse(obj), unparse(parse(unparse(obj)))) + + def test_simple_cdata(self): + obj = {'a': 'b'} + self.assertEqual(obj, parse(unparse(obj))) + self.assertEqual(unparse(obj), unparse(parse(unparse(obj)))) + + def test_cdata(self): + obj = {'a': {'#text': 'y'}} + self.assertEqual(obj, parse(unparse(obj), force_cdata=True)) + self.assertEqual(unparse(obj), unparse(parse(unparse(obj)))) + + def test_attrib(self): + obj = {'a': {'@href': 'x'}} + self.assertEqual(obj, parse(unparse(obj))) + self.assertEqual(unparse(obj), unparse(parse(unparse(obj)))) + + def test_attrib_and_cdata(self): + obj = {'a': {'@href': 'x', '#text': 'y'}} + self.assertEqual(obj, parse(unparse(obj))) + self.assertEqual(unparse(obj), unparse(parse(unparse(obj)))) + + def test_list(self): + obj = {'a': {'b': ['1', '2', '3']}} + self.assertEqual(obj, parse(unparse(obj))) + self.assertEqual(unparse(obj), unparse(parse(unparse(obj)))) + + def test_generator(self): + obj = {'a': {'b': ['1', '2', '3']}} + + def lazy_obj(): + return {'a': {'b': (i for i in ('1', '2', '3'))}} + self.assertEqual(obj, parse(unparse(lazy_obj()))) + self.assertEqual(unparse(lazy_obj()), + unparse(parse(unparse(lazy_obj())))) + + def test_no_root(self): + self.assertRaises(ValueError, unparse, {}) + + def test_multiple_roots(self): + self.assertRaises(ValueError, unparse, {'a': '1', 'b': '2'}) + self.assertRaises(ValueError, unparse, {'a': ['1', '2', '3']}) + + def test_no_root_nofulldoc(self): + self.assertEqual(unparse({}, full_document=False), '') + + def test_multiple_roots_nofulldoc(self): + obj = OrderedDict((('a', 1), ('b', 2))) + xml = unparse(obj, full_document=False) + self.assertEqual(xml, '12') + obj = {'a': [1, 2]} + xml = unparse(obj, full_document=False) + self.assertEqual(xml, '12') + + def test_nested(self): + obj = {'a': {'b': '1', 'c': '2'}} + self.assertEqual(obj, parse(unparse(obj))) + self.assertEqual(unparse(obj), unparse(parse(unparse(obj)))) + obj = {'a': {'b': {'c': {'@a': 'x', '#text': 'y'}}}} + self.assertEqual(obj, parse(unparse(obj))) + self.assertEqual(unparse(obj), unparse(parse(unparse(obj)))) + + def test_semistructured(self): + xml = 'abcefg' + self.assertEqual(_strip(unparse(parse(xml))), + 'abcefg') + + def test_preprocessor(self): + obj = {'a': OrderedDict((('b:int', [1, 2]), ('b', 'c')))} + + def p(key, value): + try: + key, _ = key.split(':') + except ValueError: + pass + return key, value + + self.assertEqual(_strip(unparse(obj, preprocessor=p)), + '12c') + + def test_preprocessor_skipkey(self): + obj = {'a': {'b': 1, 'c': 2}} + + def p(key, value): + if key == 'b': + return None + return key, value + + self.assertEqual(_strip(unparse(obj, preprocessor=p)), + '2') + + if not IS_JYTHON: + # Jython's SAX does not preserve attribute order + def test_attr_order_roundtrip(self): + xml = '' + self.assertEqual(xml, _strip(unparse(parse(xml)))) + + def test_pretty_print(self): + obj = {'a': OrderedDict(( + ('b', [{'c': [1, 2]}, 3]), + ('x', 'y'), + ))} + newl = '\n' + indent = '....' + xml = dedent('''\ + + + .... + ........1 + ........2 + .... + ....3 + ....y + ''') + self.assertEqual(xml, unparse(obj, pretty=True, + newl=newl, indent=indent)) + + def test_encoding(self): + try: + value = unichr(39321) + except NameError: + value = chr(39321) + obj = {'a': value} + utf8doc = unparse(obj, encoding='utf-8') + latin1doc = unparse(obj, encoding='iso-8859-1') + self.assertEqual(parse(utf8doc), parse(latin1doc)) + self.assertEqual(parse(utf8doc), obj) + + def test_fulldoc(self): + xml_declaration_re = re.compile( + '^' + re.escape('')) + self.assertTrue(xml_declaration_re.match(unparse({'a': 1}))) + self.assertFalse( + xml_declaration_re.match(unparse({'a': 1}, full_document=False))) + + def test_non_string_value(self): + obj = {'a': 1} + self.assertEqual('1', _strip(unparse(obj))) + + def test_non_string_attr(self): + obj = {'a': {'@attr': 1}} + self.assertEqual('', _strip(unparse(obj))) + + def test_short_empty_elements(self): + if sys.version_info[0] < 3: + return + obj = {'a': None} + self.assertEqual('', _strip(unparse(obj, short_empty_elements=True))) + + def test_namespace_support(self): + obj = OrderedDict(( + ('http://defaultns.com/:root', OrderedDict(( + ('@xmlns', OrderedDict(( + ('', 'http://defaultns.com/'), + ('a', 'http://a.com/'), + ('b', 'http://b.com/'), + ))), + ('http://defaultns.com/:x', OrderedDict(( + ('@http://a.com/:attr', 'val'), + ('#text', '1'), + ))), + ('http://a.com/:y', '2'), + ('http://b.com/:z', '3'), + ))), + )) + ns = { + 'http://defaultns.com/': '', + 'http://a.com/': 'a', + 'http://b.com/': 'b', + } + + expected_xml = ''' +123''' + xml = unparse(obj, namespaces=ns) + + self.assertEqual(xml, expected_xml) + + def test_boolean_unparse(self): + expected_xml = '\ntrue' + xml = unparse(dict(x=True)) + self.assertEqual(xml, expected_xml) + + expected_xml = '\nfalse' + xml = unparse(dict(x=False)) + self.assertEqual(xml, expected_xml) diff --git a/contrib/python/xmltodict/py2/tests/test_xmltodict.py b/contrib/python/xmltodict/py2/tests/test_xmltodict.py new file mode 100644 index 00000000000..d778816817f --- /dev/null +++ b/contrib/python/xmltodict/py2/tests/test_xmltodict.py @@ -0,0 +1,382 @@ +from xmltodict import parse, ParsingInterrupted +import unittest + +try: + from io import BytesIO as StringIO +except ImportError: + from xmltodict import StringIO + +from xml.parsers.expat import ParserCreate +from xml.parsers import expat + + +def _encode(s): + try: + return bytes(s, 'ascii') + except (NameError, TypeError): + return s + + +class XMLToDictTestCase(unittest.TestCase): + + def test_string_vs_file(self): + xml = 'data' + self.assertEqual(parse(xml), + parse(StringIO(_encode(xml)))) + + def test_minimal(self): + self.assertEqual(parse(''), + {'a': None}) + self.assertEqual(parse('', force_cdata=True), + {'a': None}) + + def test_simple(self): + self.assertEqual(parse('data'), + {'a': 'data'}) + + def test_force_cdata(self): + self.assertEqual(parse('data', force_cdata=True), + {'a': {'#text': 'data'}}) + + def test_custom_cdata(self): + self.assertEqual(parse('data', + force_cdata=True, + cdata_key='_CDATA_'), + {'a': {'_CDATA_': 'data'}}) + + def test_list(self): + self.assertEqual(parse('123'), + {'a': {'b': ['1', '2', '3']}}) + + def test_attrib(self): + self.assertEqual(parse(''), + {'a': {'@href': 'xyz'}}) + + def test_skip_attrib(self): + self.assertEqual(parse('', xml_attribs=False), + {'a': None}) + + def test_custom_attrib(self): + self.assertEqual(parse('', + attr_prefix='!'), + {'a': {'!href': 'xyz'}}) + + def test_attrib_and_cdata(self): + self.assertEqual(parse('123'), + {'a': {'@href': 'xyz', '#text': '123'}}) + + def test_semi_structured(self): + self.assertEqual(parse('abcdef'), + {'a': {'b': None, '#text': 'abcdef'}}) + self.assertEqual(parse('abcdef', + cdata_separator='\n'), + {'a': {'b': None, '#text': 'abc\ndef'}}) + + def test_nested_semi_structured(self): + self.assertEqual(parse('abc123456def'), + {'a': {'#text': 'abcdef', 'b': { + '#text': '123456', 'c': None}}}) + + def test_skip_whitespace(self): + xml = """ + + + + + + + + + hello + + """ + self.assertEqual( + parse(xml), + {'root': {'emptya': None, + 'emptyb': {'@attr': 'attrvalue'}, + 'value': 'hello'}}) + + def test_keep_whitespace(self): + xml = " " + self.assertEqual(parse(xml), dict(root=None)) + self.assertEqual(parse(xml, strip_whitespace=False), + dict(root=' ')) + + def test_streaming(self): + def cb(path, item): + cb.count += 1 + self.assertEqual(path, [('a', {'x': 'y'}), ('b', None)]) + self.assertEqual(item, str(cb.count)) + return True + cb.count = 0 + parse('123', + item_depth=2, item_callback=cb) + self.assertEqual(cb.count, 3) + + def test_streaming_interrupt(self): + cb = lambda path, item: False + self.assertRaises(ParsingInterrupted, + parse, 'x', + item_depth=1, item_callback=cb) + + def test_postprocessor(self): + def postprocessor(path, key, value): + try: + return key + ':int', int(value) + except (ValueError, TypeError): + return key, value + self.assertEqual({'a': {'b:int': [1, 2], 'b': 'x'}}, + parse('12x', + postprocessor=postprocessor)) + + def test_postprocessor_attribute(self): + def postprocessor(path, key, value): + try: + return key + ':int', int(value) + except (ValueError, TypeError): + return key, value + self.assertEqual({'a': {'@b:int': 1}}, + parse('', + postprocessor=postprocessor)) + + def test_postprocessor_skip(self): + def postprocessor(path, key, value): + if key == 'b': + value = int(value) + if value == 3: + return None + return key, value + self.assertEqual({'a': {'b': [1, 2]}}, + parse('123', + postprocessor=postprocessor)) + + def test_unicode(self): + try: + value = unichr(39321) + except NameError: + value = chr(39321) + self.assertEqual({'a': value}, + parse('%s' % value)) + + def test_encoded_string(self): + try: + value = unichr(39321) + except NameError: + value = chr(39321) + xml = '%s' % value + self.assertEqual(parse(xml), + parse(xml.encode('utf-8'))) + + def test_namespace_support(self): + xml = """ + + 1 + 2 + 3 + + """ + d = { + 'http://defaultns.com/:root': { + 'http://defaultns.com/:x': { + '@xmlns': { + '': 'http://defaultns.com/', + 'a': 'http://a.com/', + 'b': 'http://b.com/', + }, + '@http://a.com/:attr': 'val', + '#text': '1', + }, + 'http://a.com/:y': '2', + 'http://b.com/:z': '3', + } + } + res = parse(xml, process_namespaces=True) + self.assertEqual(res, d) + + def test_namespace_collapse(self): + xml = """ + + 1 + 2 + 3 + + """ + namespaces = { + 'http://defaultns.com/': '', + 'http://a.com/': 'ns_a', + } + d = { + 'root': { + 'x': { + '@xmlns': { + '': 'http://defaultns.com/', + 'a': 'http://a.com/', + 'b': 'http://b.com/', + }, + '@ns_a:attr': 'val', + '#text': '1', + }, + 'ns_a:y': '2', + 'http://b.com/:z': '3', + }, + } + res = parse(xml, process_namespaces=True, namespaces=namespaces) + self.assertEqual(res, d) + + def test_namespace_ignore(self): + xml = """ + + 1 + 2 + 3 + + """ + d = { + 'root': { + '@xmlns': 'http://defaultns.com/', + '@xmlns:a': 'http://a.com/', + '@xmlns:b': 'http://b.com/', + 'x': '1', + 'a:y': '2', + 'b:z': '3', + }, + } + self.assertEqual(parse(xml), d) + + def test_force_list_basic(self): + xml = """ + + + server1 + os1 + + + """ + expectedResult = { + 'servers': { + 'server': [ + { + 'name': 'server1', + 'os': 'os1', + }, + ], + } + } + self.assertEqual(parse(xml, force_list=('server',)), expectedResult) + + def test_force_list_callable(self): + xml = """ + + + + server1 + os1 + + + + + + + """ + + def force_list(path, key, value): + """Only return True for servers/server, but not for skip/server.""" + if key != 'server': + return False + return path and path[-1][0] == 'servers' + + expectedResult = { + 'config': { + 'servers': { + 'server': [ + { + 'name': 'server1', + 'os': 'os1', + }, + ], + }, + 'skip': { + 'server': None, + }, + }, + } + self.assertEqual(parse(xml, force_list=force_list, dict_constructor=dict), expectedResult) + + def test_disable_entities_true_ignores_xmlbomb(self): + xml = """ + + + + ]> + &c; + """ + expectedResult = {'bomb': None} + try: + parse_attempt = parse(xml, disable_entities=True) + except expat.ExpatError: + self.assertTrue(True) + else: + self.assertEqual(parse_attempt, expectedResult) + + def test_disable_entities_false_returns_xmlbomb(self): + xml = """ + + + + ]> + &c; + """ + bomb = "1234567890" * 64 + expectedResult = {'bomb': bomb} + self.assertEqual(parse(xml, disable_entities=False), expectedResult) + + def test_disable_entities_true_ignores_external_dtd(self): + xml = """ + + ]> + ⅇ + """ + expectedResult = {'root': None} + try: + parse_attempt = parse(xml, disable_entities=True) + except expat.ExpatError: + self.assertTrue(True) + else: + self.assertEqual(parse_attempt, expectedResult) + + def test_disable_entities_true_attempts_external_dtd(self): + xml = """ + + ]> + ⅇ + """ + + def raising_external_ref_handler(*args, **kwargs): + parser = ParserCreate(*args, **kwargs) + parser.ExternalEntityRefHandler = lambda *x: 0 + try: + feature = "http://apache.org/xml/features/disallow-doctype-decl" + parser._reader.setFeature(feature, True) + except AttributeError: + pass + return parser + expat.ParserCreate = raising_external_ref_handler + # Using this try/catch because a TypeError is thrown before + # the ExpatError, and Python 2.6 is confused by that. + try: + parse(xml, disable_entities=False, expat=expat) + except expat.ExpatError: + self.assertTrue(True) + else: + self.assertTrue(False) + expat.ParserCreate = ParserCreate diff --git a/contrib/python/xmltodict/py2/xmltodict.py b/contrib/python/xmltodict/py2/xmltodict.py new file mode 100644 index 00000000000..d6dbcd7a700 --- /dev/null +++ b/contrib/python/xmltodict/py2/xmltodict.py @@ -0,0 +1,488 @@ +#!/usr/bin/env python +"Makes working with XML feel like you are working with JSON" + +try: + from defusedexpat import pyexpat as expat +except ImportError: + from xml.parsers import expat +from xml.sax.saxutils import XMLGenerator +from xml.sax.xmlreader import AttributesImpl +try: # pragma no cover + from cStringIO import StringIO +except ImportError: # pragma no cover + try: + from StringIO import StringIO + except ImportError: + from io import StringIO + +from collections import OrderedDict + +try: # pragma no cover + _basestring = basestring +except NameError: # pragma no cover + _basestring = str +try: # pragma no cover + _unicode = unicode +except NameError: # pragma no cover + _unicode = str + +__author__ = 'Martin Blech' +__version__ = '0.12.0' +__license__ = 'MIT' + + +class ParsingInterrupted(Exception): + pass + + +class _DictSAXHandler(object): + def __init__(self, + item_depth=0, + item_callback=lambda *args: True, + xml_attribs=True, + attr_prefix='@', + cdata_key='#text', + force_cdata=False, + cdata_separator='', + postprocessor=None, + dict_constructor=OrderedDict, + strip_whitespace=True, + namespace_separator=':', + namespaces=None, + force_list=None): + self.path = [] + self.stack = [] + self.data = [] + self.item = None + self.item_depth = item_depth + self.xml_attribs = xml_attribs + self.item_callback = item_callback + self.attr_prefix = attr_prefix + self.cdata_key = cdata_key + self.force_cdata = force_cdata + self.cdata_separator = cdata_separator + self.postprocessor = postprocessor + self.dict_constructor = dict_constructor + self.strip_whitespace = strip_whitespace + self.namespace_separator = namespace_separator + self.namespaces = namespaces + self.namespace_declarations = OrderedDict() + self.force_list = force_list + + def _build_name(self, full_name): + if not self.namespaces: + return full_name + i = full_name.rfind(self.namespace_separator) + if i == -1: + return full_name + namespace, name = full_name[:i], full_name[i+1:] + short_namespace = self.namespaces.get(namespace, namespace) + if not short_namespace: + return name + else: + return self.namespace_separator.join((short_namespace, name)) + + def _attrs_to_dict(self, attrs): + if isinstance(attrs, dict): + return attrs + return self.dict_constructor(zip(attrs[0::2], attrs[1::2])) + + def startNamespaceDecl(self, prefix, uri): + self.namespace_declarations[prefix or ''] = uri + + def startElement(self, full_name, attrs): + name = self._build_name(full_name) + attrs = self._attrs_to_dict(attrs) + if attrs and self.namespace_declarations: + attrs['xmlns'] = self.namespace_declarations + self.namespace_declarations = OrderedDict() + self.path.append((name, attrs or None)) + if len(self.path) > self.item_depth: + self.stack.append((self.item, self.data)) + if self.xml_attribs: + attr_entries = [] + for key, value in attrs.items(): + key = self.attr_prefix+self._build_name(key) + if self.postprocessor: + entry = self.postprocessor(self.path, key, value) + else: + entry = (key, value) + if entry: + attr_entries.append(entry) + attrs = self.dict_constructor(attr_entries) + else: + attrs = None + self.item = attrs or None + self.data = [] + + def endElement(self, full_name): + name = self._build_name(full_name) + if len(self.path) == self.item_depth: + item = self.item + if item is None: + item = (None if not self.data + else self.cdata_separator.join(self.data)) + + should_continue = self.item_callback(self.path, item) + if not should_continue: + raise ParsingInterrupted() + if len(self.stack): + data = (None if not self.data + else self.cdata_separator.join(self.data)) + item = self.item + self.item, self.data = self.stack.pop() + if self.strip_whitespace and data: + data = data.strip() or None + if data and self.force_cdata and item is None: + item = self.dict_constructor() + if item is not None: + if data: + self.push_data(item, self.cdata_key, data) + self.item = self.push_data(self.item, name, item) + else: + self.item = self.push_data(self.item, name, data) + else: + self.item = None + self.data = [] + self.path.pop() + + def characters(self, data): + if not self.data: + self.data = [data] + else: + self.data.append(data) + + def push_data(self, item, key, data): + if self.postprocessor is not None: + result = self.postprocessor(self.path, key, data) + if result is None: + return item + key, data = result + if item is None: + item = self.dict_constructor() + try: + value = item[key] + if isinstance(value, list): + value.append(data) + else: + item[key] = [value, data] + except KeyError: + if self._should_force_list(key, data): + item[key] = [data] + else: + item[key] = data + return item + + def _should_force_list(self, key, value): + if not self.force_list: + return False + if isinstance(self.force_list, bool): + return self.force_list + try: + return key in self.force_list + except TypeError: + return self.force_list(self.path[:-1], key, value) + + +def parse(xml_input, encoding=None, expat=expat, process_namespaces=False, + namespace_separator=':', disable_entities=True, **kwargs): + """Parse the given XML input and convert it into a dictionary. + + `xml_input` can either be a `string` or a file-like object. + + If `xml_attribs` is `True`, element attributes are put in the dictionary + among regular child elements, using `@` as a prefix to avoid collisions. If + set to `False`, they are just ignored. + + Simple example:: + + >>> import xmltodict + >>> doc = xmltodict.parse(\"\"\" + ... + ... 1 + ... 2 + ... + ... \"\"\") + >>> doc['a']['@prop'] + u'x' + >>> doc['a']['b'] + [u'1', u'2'] + + If `item_depth` is `0`, the function returns a dictionary for the root + element (default behavior). Otherwise, it calls `item_callback` every time + an item at the specified depth is found and returns `None` in the end + (streaming mode). + + The callback function receives two parameters: the `path` from the document + root to the item (name-attribs pairs), and the `item` (dict). If the + callback's return value is false-ish, parsing will be stopped with the + :class:`ParsingInterrupted` exception. + + Streaming example:: + + >>> def handle(path, item): + ... print('path:%s item:%s' % (path, item)) + ... return True + ... + >>> xmltodict.parse(\"\"\" + ... + ... 1 + ... 2 + ... \"\"\", item_depth=2, item_callback=handle) + path:[(u'a', {u'prop': u'x'}), (u'b', None)] item:1 + path:[(u'a', {u'prop': u'x'}), (u'b', None)] item:2 + + The optional argument `postprocessor` is a function that takes `path`, + `key` and `value` as positional arguments and returns a new `(key, value)` + pair where both `key` and `value` may have changed. Usage example:: + + >>> def postprocessor(path, key, value): + ... try: + ... return key + ':int', int(value) + ... except (ValueError, TypeError): + ... return key, value + >>> xmltodict.parse('12x', + ... postprocessor=postprocessor) + OrderedDict([(u'a', OrderedDict([(u'b:int', [1, 2]), (u'b', u'x')]))]) + + You can pass an alternate version of `expat` (such as `defusedexpat`) by + using the `expat` parameter. E.g: + + >>> import defusedexpat + >>> xmltodict.parse('hello', expat=defusedexpat.pyexpat) + OrderedDict([(u'a', u'hello')]) + + You can use the force_list argument to force lists to be created even + when there is only a single child of a given level of hierarchy. The + force_list argument is a tuple of keys. If the key for a given level + of hierarchy is in the force_list argument, that level of hierarchy + will have a list as a child (even if there is only one sub-element). + The index_keys operation takes precendence over this. This is applied + after any user-supplied postprocessor has already run. + + For example, given this input: + + + host1 + Linux + + + em0 + 10.0.0.1 + + + + + + If called with force_list=('interface',), it will produce + this dictionary: + {'servers': + {'server': + {'name': 'host1', + 'os': 'Linux'}, + 'interfaces': + {'interface': + [ {'name': 'em0', 'ip_address': '10.0.0.1' } ] } } } + + `force_list` can also be a callable that receives `path`, `key` and + `value`. This is helpful in cases where the logic that decides whether + a list should be forced is more complex. + """ + handler = _DictSAXHandler(namespace_separator=namespace_separator, + **kwargs) + if isinstance(xml_input, _unicode): + if not encoding: + encoding = 'utf-8' + xml_input = xml_input.encode(encoding) + if not process_namespaces: + namespace_separator = None + parser = expat.ParserCreate( + encoding, + namespace_separator + ) + try: + parser.ordered_attributes = True + except AttributeError: + # Jython's expat does not support ordered_attributes + pass + parser.StartNamespaceDeclHandler = handler.startNamespaceDecl + parser.StartElementHandler = handler.startElement + parser.EndElementHandler = handler.endElement + parser.CharacterDataHandler = handler.characters + parser.buffer_text = True + if disable_entities: + try: + # Attempt to disable DTD in Jython's expat parser (Xerces-J). + feature = "http://apache.org/xml/features/disallow-doctype-decl" + parser._reader.setFeature(feature, True) + except AttributeError: + # For CPython / expat parser. + # Anything not handled ends up here and entities aren't expanded. + parser.DefaultHandler = lambda x: None + # Expects an integer return; zero means failure -> expat.ExpatError. + parser.ExternalEntityRefHandler = lambda *x: 1 + if hasattr(xml_input, 'read'): + parser.ParseFile(xml_input) + else: + parser.Parse(xml_input, True) + return handler.item + + +def _process_namespace(name, namespaces, ns_sep=':', attr_prefix='@'): + if not namespaces: + return name + try: + ns, name = name.rsplit(ns_sep, 1) + except ValueError: + pass + else: + ns_res = namespaces.get(ns.strip(attr_prefix)) + name = '{}{}{}{}'.format( + attr_prefix if ns.startswith(attr_prefix) else '', + ns_res, ns_sep, name) if ns_res else name + return name + + +def _emit(key, value, content_handler, + attr_prefix='@', + cdata_key='#text', + depth=0, + preprocessor=None, + pretty=False, + newl='\n', + indent='\t', + namespace_separator=':', + namespaces=None, + full_document=True): + key = _process_namespace(key, namespaces, namespace_separator, attr_prefix) + if preprocessor is not None: + result = preprocessor(key, value) + if result is None: + return + key, value = result + if (not hasattr(value, '__iter__') + or isinstance(value, _basestring) + or isinstance(value, dict)): + value = [value] + for index, v in enumerate(value): + if full_document and depth == 0 and index > 0: + raise ValueError('document with multiple roots') + if v is None: + v = OrderedDict() + elif isinstance(v, bool): + if v: + v = _unicode('true') + else: + v = _unicode('false') + elif not isinstance(v, dict): + v = _unicode(v) + if isinstance(v, _basestring): + v = OrderedDict(((cdata_key, v),)) + cdata = None + attrs = OrderedDict() + children = [] + for ik, iv in v.items(): + if ik == cdata_key: + cdata = iv + continue + if ik.startswith(attr_prefix): + ik = _process_namespace(ik, namespaces, namespace_separator, + attr_prefix) + if ik == '@xmlns' and isinstance(iv, dict): + for k, v in iv.items(): + attr = 'xmlns{}'.format(':{}'.format(k) if k else '') + attrs[attr] = _unicode(v) + continue + if not isinstance(iv, _unicode): + iv = _unicode(iv) + attrs[ik[len(attr_prefix):]] = iv + continue + children.append((ik, iv)) + if pretty: + content_handler.ignorableWhitespace(depth * indent) + content_handler.startElement(key, AttributesImpl(attrs)) + if pretty and children: + content_handler.ignorableWhitespace(newl) + for child_key, child_value in children: + _emit(child_key, child_value, content_handler, + attr_prefix, cdata_key, depth+1, preprocessor, + pretty, newl, indent, namespaces=namespaces, + namespace_separator=namespace_separator) + if cdata is not None: + content_handler.characters(cdata) + if pretty and children: + content_handler.ignorableWhitespace(depth * indent) + content_handler.endElement(key) + if pretty and depth: + content_handler.ignorableWhitespace(newl) + + +def unparse(input_dict, output=None, encoding='utf-8', full_document=True, + short_empty_elements=False, + **kwargs): + """Emit an XML document for the given `input_dict` (reverse of `parse`). + + The resulting XML document is returned as a string, but if `output` (a + file-like object) is specified, it is written there instead. + + Dictionary keys prefixed with `attr_prefix` (default=`'@'`) are interpreted + as XML node attributes, whereas keys equal to `cdata_key` + (default=`'#text'`) are treated as character data. + + The `pretty` parameter (default=`False`) enables pretty-printing. In this + mode, lines are terminated with `'\n'` and indented with `'\t'`, but this + can be customized with the `newl` and `indent` parameters. + + """ + if full_document and len(input_dict) != 1: + raise ValueError('Document must have exactly one root.') + must_return = False + if output is None: + output = StringIO() + must_return = True + if short_empty_elements: + content_handler = XMLGenerator(output, encoding, True) + else: + content_handler = XMLGenerator(output, encoding) + if full_document: + content_handler.startDocument() + for key, value in input_dict.items(): + _emit(key, value, content_handler, full_document=full_document, + **kwargs) + if full_document: + content_handler.endDocument() + if must_return: + value = output.getvalue() + try: # pragma no cover + value = value.decode(encoding) + except AttributeError: # pragma no cover + pass + return value + + +if __name__ == '__main__': # pragma: no cover + import sys + import marshal + try: + stdin = sys.stdin.buffer + stdout = sys.stdout.buffer + except AttributeError: + stdin = sys.stdin + stdout = sys.stdout + + (item_depth,) = sys.argv[1:] + item_depth = int(item_depth) + + def handle_item(path, item): + marshal.dump((path, item), stdout) + return True + + try: + root = parse(stdin, + item_depth=item_depth, + item_callback=handle_item, + dict_constructor=dict) + if item_depth == 0: + handle_item([], root) + except KeyboardInterrupt: + pass diff --git a/contrib/python/xmltodict/py3/.dist-info/METADATA b/contrib/python/xmltodict/py3/.dist-info/METADATA new file mode 100644 index 00000000000..73ad4cc22f8 --- /dev/null +++ b/contrib/python/xmltodict/py3/.dist-info/METADATA @@ -0,0 +1,234 @@ +Metadata-Version: 2.1 +Name: xmltodict +Version: 0.12.0 +Summary: Makes working with XML feel like you are working with JSON +Home-page: https://github.com/martinblech/xmltodict +Author: Martin Blech +Author-email: martinblech@gmail.com +License: MIT +Platform: all +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: Implementation :: Jython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Text Processing :: Markup :: XML +Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* +Description-Content-Type: text/markdown + +# xmltodict + +`xmltodict` is a Python module that makes working with XML feel like you are working with [JSON](http://docs.python.org/library/json.html), as in this ["spec"](http://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html): + +[![Build Status](https://secure.travis-ci.org/martinblech/xmltodict.svg)](http://travis-ci.org/martinblech/xmltodict) + +```python +>>> print(json.dumps(xmltodict.parse(""" +... +... +... elements +... more elements +... +... +... element as well +... +... +... """), indent=4)) +{ + "mydocument": { + "@has": "an attribute", + "and": { + "many": [ + "elements", + "more elements" + ] + }, + "plus": { + "@a": "complex", + "#text": "element as well" + } + } +} +``` + +## Namespace support + +By default, `xmltodict` does no XML namespace processing (it just treats namespace declarations as regular node attributes), but passing `process_namespaces=True` will make it expand namespaces for you: + +```python +>>> xml = """ +... +... 1 +... 2 +... 3 +... +... """ +>>> xmltodict.parse(xml, process_namespaces=True) == { +... 'http://defaultns.com/:root': { +... 'http://defaultns.com/:x': '1', +... 'http://a.com/:y': '2', +... 'http://b.com/:z': '3', +... } +... } +True +``` + +It also lets you collapse certain namespaces to shorthand prefixes, or skip them altogether: + +```python +>>> namespaces = { +... 'http://defaultns.com/': None, # skip this namespace +... 'http://a.com/': 'ns_a', # collapse "http://a.com/" -> "ns_a" +... } +>>> xmltodict.parse(xml, process_namespaces=True, namespaces=namespaces) == { +... 'root': { +... 'x': '1', +... 'ns_a:y': '2', +... 'http://b.com/:z': '3', +... }, +... } +True +``` + +## Streaming mode + +`xmltodict` is very fast ([Expat](http://docs.python.org/library/pyexpat.html)-based) and has a streaming mode with a small memory footprint, suitable for big XML dumps like [Discogs](http://discogs.com/data/) or [Wikipedia](http://dumps.wikimedia.org/): + +```python +>>> def handle_artist(_, artist): +... print(artist['name']) +... return True +>>> +>>> xmltodict.parse(GzipFile('discogs_artists.xml.gz'), +... item_depth=2, item_callback=handle_artist) +A Perfect Circle +Fantômas +King Crimson +Chris Potter +... +``` + +It can also be used from the command line to pipe objects to a script like this: + +```python +import sys, marshal +while True: + _, article = marshal.load(sys.stdin) + print(article['title']) +``` + +```sh +$ bunzip2 enwiki-pages-articles.xml.bz2 | xmltodict.py 2 | myscript.py +AccessibleComputing +Anarchism +AfghanistanHistory +AfghanistanGeography +AfghanistanPeople +AfghanistanCommunications +Autism +... +``` + +Or just cache the dicts so you don't have to parse that big XML file again. You do this only once: + +```sh +$ bunzip2 enwiki-pages-articles.xml.bz2 | xmltodict.py 2 | gzip > enwiki.dicts.gz +``` + +And you reuse the dicts with every script that needs them: + +```sh +$ gunzip enwiki.dicts.gz | script1.py +$ gunzip enwiki.dicts.gz | script2.py +... +``` + +## Roundtripping + +You can also convert in the other direction, using the `unparse()` method: + +```python +>>> mydict = { +... 'response': { +... 'status': 'good', +... 'last_updated': '2014-02-16T23:10:12Z', +... } +... } +>>> print(unparse(mydict, pretty=True)) + + + good + 2014-02-16T23:10:12Z + +``` + +Text values for nodes can be specified with the `cdata_key` key in the python dict, while node properties can be specified with the `attr_prefix` prefixed to the key name in the python dict. The default value for `attr_prefix` is `@` and the default value for `cdata_key` is `#text`. + +```python +>>> import xmltodict +>>> +>>> mydict = { +... 'text': { +... '@color':'red', +... '@stroke':'2', +... '#text':'This is a test' +... } +... } +>>> print(xmltodict.unparse(mydict, pretty=True)) + +This is a test +``` + +## Ok, how do I get it? + +### Using pypi + +You just need to + +```sh +$ pip install xmltodict +``` + +### RPM-based distro (Fedora, RHEL, …) + +There is an [official Fedora package for xmltodict](https://apps.fedoraproject.org/packages/python-xmltodict). + +```sh +$ sudo yum install python-xmltodict +``` + +### Arch Linux + +There is an [official Arch Linux package for xmltodict](https://www.archlinux.org/packages/community/any/python-xmltodict/). + +```sh +$ sudo pacman -S python-xmltodict +``` + +### Debian-based distro (Debian, Ubuntu, …) + +There is an [official Debian package for xmltodict](https://tracker.debian.org/pkg/python-xmltodict). + +```sh +$ sudo apt install python-xmltodict +``` + +### FreeBSD + +There is an [official FreeBSD port for xmltodict](https://svnweb.freebsd.org/ports/head/devel/py-xmltodict/). + +```sh +$ pkg install py36-xmltodict +``` + + diff --git a/contrib/python/xmltodict/py3/.dist-info/top_level.txt b/contrib/python/xmltodict/py3/.dist-info/top_level.txt new file mode 100644 index 00000000000..a0d4bb760b6 --- /dev/null +++ b/contrib/python/xmltodict/py3/.dist-info/top_level.txt @@ -0,0 +1 @@ +xmltodict diff --git a/contrib/python/xmltodict/py3/LICENSE b/contrib/python/xmltodict/py3/LICENSE new file mode 100644 index 00000000000..a462778cff0 --- /dev/null +++ b/contrib/python/xmltodict/py3/LICENSE @@ -0,0 +1,7 @@ +Copyright (C) 2012 Martin Blech and individual contributors. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/contrib/python/xmltodict/py3/README.md b/contrib/python/xmltodict/py3/README.md new file mode 100644 index 00000000000..59177da90b7 --- /dev/null +++ b/contrib/python/xmltodict/py3/README.md @@ -0,0 +1,206 @@ +# xmltodict + +`xmltodict` is a Python module that makes working with XML feel like you are working with [JSON](http://docs.python.org/library/json.html), as in this ["spec"](http://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html): + +[![Build Status](https://secure.travis-ci.org/martinblech/xmltodict.svg)](http://travis-ci.org/martinblech/xmltodict) + +```python +>>> print(json.dumps(xmltodict.parse(""" +... +... +... elements +... more elements +... +... +... element as well +... +... +... """), indent=4)) +{ + "mydocument": { + "@has": "an attribute", + "and": { + "many": [ + "elements", + "more elements" + ] + }, + "plus": { + "@a": "complex", + "#text": "element as well" + } + } +} +``` + +## Namespace support + +By default, `xmltodict` does no XML namespace processing (it just treats namespace declarations as regular node attributes), but passing `process_namespaces=True` will make it expand namespaces for you: + +```python +>>> xml = """ +... +... 1 +... 2 +... 3 +... +... """ +>>> xmltodict.parse(xml, process_namespaces=True) == { +... 'http://defaultns.com/:root': { +... 'http://defaultns.com/:x': '1', +... 'http://a.com/:y': '2', +... 'http://b.com/:z': '3', +... } +... } +True +``` + +It also lets you collapse certain namespaces to shorthand prefixes, or skip them altogether: + +```python +>>> namespaces = { +... 'http://defaultns.com/': None, # skip this namespace +... 'http://a.com/': 'ns_a', # collapse "http://a.com/" -> "ns_a" +... } +>>> xmltodict.parse(xml, process_namespaces=True, namespaces=namespaces) == { +... 'root': { +... 'x': '1', +... 'ns_a:y': '2', +... 'http://b.com/:z': '3', +... }, +... } +True +``` + +## Streaming mode + +`xmltodict` is very fast ([Expat](http://docs.python.org/library/pyexpat.html)-based) and has a streaming mode with a small memory footprint, suitable for big XML dumps like [Discogs](http://discogs.com/data/) or [Wikipedia](http://dumps.wikimedia.org/): + +```python +>>> def handle_artist(_, artist): +... print(artist['name']) +... return True +>>> +>>> xmltodict.parse(GzipFile('discogs_artists.xml.gz'), +... item_depth=2, item_callback=handle_artist) +A Perfect Circle +Fantômas +King Crimson +Chris Potter +... +``` + +It can also be used from the command line to pipe objects to a script like this: + +```python +import sys, marshal +while True: + _, article = marshal.load(sys.stdin) + print(article['title']) +``` + +```sh +$ bunzip2 enwiki-pages-articles.xml.bz2 | xmltodict.py 2 | myscript.py +AccessibleComputing +Anarchism +AfghanistanHistory +AfghanistanGeography +AfghanistanPeople +AfghanistanCommunications +Autism +... +``` + +Or just cache the dicts so you don't have to parse that big XML file again. You do this only once: + +```sh +$ bunzip2 enwiki-pages-articles.xml.bz2 | xmltodict.py 2 | gzip > enwiki.dicts.gz +``` + +And you reuse the dicts with every script that needs them: + +```sh +$ gunzip enwiki.dicts.gz | script1.py +$ gunzip enwiki.dicts.gz | script2.py +... +``` + +## Roundtripping + +You can also convert in the other direction, using the `unparse()` method: + +```python +>>> mydict = { +... 'response': { +... 'status': 'good', +... 'last_updated': '2014-02-16T23:10:12Z', +... } +... } +>>> print(unparse(mydict, pretty=True)) + + + good + 2014-02-16T23:10:12Z + +``` + +Text values for nodes can be specified with the `cdata_key` key in the python dict, while node properties can be specified with the `attr_prefix` prefixed to the key name in the python dict. The default value for `attr_prefix` is `@` and the default value for `cdata_key` is `#text`. + +```python +>>> import xmltodict +>>> +>>> mydict = { +... 'text': { +... '@color':'red', +... '@stroke':'2', +... '#text':'This is a test' +... } +... } +>>> print(xmltodict.unparse(mydict, pretty=True)) + +This is a test +``` + +## Ok, how do I get it? + +### Using pypi + +You just need to + +```sh +$ pip install xmltodict +``` + +### RPM-based distro (Fedora, RHEL, …) + +There is an [official Fedora package for xmltodict](https://apps.fedoraproject.org/packages/python-xmltodict). + +```sh +$ sudo yum install python-xmltodict +``` + +### Arch Linux + +There is an [official Arch Linux package for xmltodict](https://www.archlinux.org/packages/community/any/python-xmltodict/). + +```sh +$ sudo pacman -S python-xmltodict +``` + +### Debian-based distro (Debian, Ubuntu, …) + +There is an [official Debian package for xmltodict](https://tracker.debian.org/pkg/python-xmltodict). + +```sh +$ sudo apt install python-xmltodict +``` + +### FreeBSD + +There is an [official FreeBSD port for xmltodict](https://svnweb.freebsd.org/ports/head/devel/py-xmltodict/). + +```sh +$ pkg install py36-xmltodict +``` diff --git a/contrib/python/xmltodict/py3/tests/test_dicttoxml.py b/contrib/python/xmltodict/py3/tests/test_dicttoxml.py new file mode 100644 index 00000000000..84fa5da871b --- /dev/null +++ b/contrib/python/xmltodict/py3/tests/test_dicttoxml.py @@ -0,0 +1,207 @@ +import sys +from xmltodict import parse, unparse +from collections import OrderedDict + +import unittest +import re +from textwrap import dedent + +IS_JYTHON = sys.platform.startswith('java') + +_HEADER_RE = re.compile(r'^[^\n]*\n') + + +def _strip(fullxml): + return _HEADER_RE.sub('', fullxml) + + +class DictToXMLTestCase(unittest.TestCase): + def test_root(self): + obj = {'a': None} + self.assertEqual(obj, parse(unparse(obj))) + self.assertEqual(unparse(obj), unparse(parse(unparse(obj)))) + + def test_simple_cdata(self): + obj = {'a': 'b'} + self.assertEqual(obj, parse(unparse(obj))) + self.assertEqual(unparse(obj), unparse(parse(unparse(obj)))) + + def test_cdata(self): + obj = {'a': {'#text': 'y'}} + self.assertEqual(obj, parse(unparse(obj), force_cdata=True)) + self.assertEqual(unparse(obj), unparse(parse(unparse(obj)))) + + def test_attrib(self): + obj = {'a': {'@href': 'x'}} + self.assertEqual(obj, parse(unparse(obj))) + self.assertEqual(unparse(obj), unparse(parse(unparse(obj)))) + + def test_attrib_and_cdata(self): + obj = {'a': {'@href': 'x', '#text': 'y'}} + self.assertEqual(obj, parse(unparse(obj))) + self.assertEqual(unparse(obj), unparse(parse(unparse(obj)))) + + def test_list(self): + obj = {'a': {'b': ['1', '2', '3']}} + self.assertEqual(obj, parse(unparse(obj))) + self.assertEqual(unparse(obj), unparse(parse(unparse(obj)))) + + def test_generator(self): + obj = {'a': {'b': ['1', '2', '3']}} + + def lazy_obj(): + return {'a': {'b': (i for i in ('1', '2', '3'))}} + self.assertEqual(obj, parse(unparse(lazy_obj()))) + self.assertEqual(unparse(lazy_obj()), + unparse(parse(unparse(lazy_obj())))) + + def test_no_root(self): + self.assertRaises(ValueError, unparse, {}) + + def test_multiple_roots(self): + self.assertRaises(ValueError, unparse, {'a': '1', 'b': '2'}) + self.assertRaises(ValueError, unparse, {'a': ['1', '2', '3']}) + + def test_no_root_nofulldoc(self): + self.assertEqual(unparse({}, full_document=False), '') + + def test_multiple_roots_nofulldoc(self): + obj = OrderedDict((('a', 1), ('b', 2))) + xml = unparse(obj, full_document=False) + self.assertEqual(xml, '12') + obj = {'a': [1, 2]} + xml = unparse(obj, full_document=False) + self.assertEqual(xml, '12') + + def test_nested(self): + obj = {'a': {'b': '1', 'c': '2'}} + self.assertEqual(obj, parse(unparse(obj))) + self.assertEqual(unparse(obj), unparse(parse(unparse(obj)))) + obj = {'a': {'b': {'c': {'@a': 'x', '#text': 'y'}}}} + self.assertEqual(obj, parse(unparse(obj))) + self.assertEqual(unparse(obj), unparse(parse(unparse(obj)))) + + def test_semistructured(self): + xml = 'abcefg' + self.assertEqual(_strip(unparse(parse(xml))), + 'abcefg') + + def test_preprocessor(self): + obj = {'a': OrderedDict((('b:int', [1, 2]), ('b', 'c')))} + + def p(key, value): + try: + key, _ = key.split(':') + except ValueError: + pass + return key, value + + self.assertEqual(_strip(unparse(obj, preprocessor=p)), + '12c') + + def test_preprocessor_skipkey(self): + obj = {'a': {'b': 1, 'c': 2}} + + def p(key, value): + if key == 'b': + return None + return key, value + + self.assertEqual(_strip(unparse(obj, preprocessor=p)), + '2') + + if not IS_JYTHON: + # Jython's SAX does not preserve attribute order + def test_attr_order_roundtrip(self): + xml = '' + self.assertEqual(xml, _strip(unparse(parse(xml)))) + + def test_pretty_print(self): + obj = {'a': OrderedDict(( + ('b', [{'c': [1, 2]}, 3]), + ('x', 'y'), + ))} + newl = '\n' + indent = '....' + xml = dedent('''\ + + + .... + ........1 + ........2 + .... + ....3 + ....y + ''') + self.assertEqual(xml, unparse(obj, pretty=True, + newl=newl, indent=indent)) + + def test_encoding(self): + try: + value = unichr(39321) + except NameError: + value = chr(39321) + obj = {'a': value} + utf8doc = unparse(obj, encoding='utf-8') + latin1doc = unparse(obj, encoding='iso-8859-1') + self.assertEqual(parse(utf8doc), parse(latin1doc)) + self.assertEqual(parse(utf8doc), obj) + + def test_fulldoc(self): + xml_declaration_re = re.compile( + '^' + re.escape('')) + self.assertTrue(xml_declaration_re.match(unparse({'a': 1}))) + self.assertFalse( + xml_declaration_re.match(unparse({'a': 1}, full_document=False))) + + def test_non_string_value(self): + obj = {'a': 1} + self.assertEqual('1', _strip(unparse(obj))) + + def test_non_string_attr(self): + obj = {'a': {'@attr': 1}} + self.assertEqual('', _strip(unparse(obj))) + + def test_short_empty_elements(self): + if sys.version_info[0] < 3: + return + obj = {'a': None} + self.assertEqual('', _strip(unparse(obj, short_empty_elements=True))) + + def test_namespace_support(self): + obj = OrderedDict(( + ('http://defaultns.com/:root', OrderedDict(( + ('@xmlns', OrderedDict(( + ('', 'http://defaultns.com/'), + ('a', 'http://a.com/'), + ('b', 'http://b.com/'), + ))), + ('http://defaultns.com/:x', OrderedDict(( + ('@http://a.com/:attr', 'val'), + ('#text', '1'), + ))), + ('http://a.com/:y', '2'), + ('http://b.com/:z', '3'), + ))), + )) + ns = { + 'http://defaultns.com/': '', + 'http://a.com/': 'a', + 'http://b.com/': 'b', + } + + expected_xml = ''' +123''' + xml = unparse(obj, namespaces=ns) + + self.assertEqual(xml, expected_xml) + + def test_boolean_unparse(self): + expected_xml = '\ntrue' + xml = unparse(dict(x=True)) + self.assertEqual(xml, expected_xml) + + expected_xml = '\nfalse' + xml = unparse(dict(x=False)) + self.assertEqual(xml, expected_xml) diff --git a/contrib/python/xmltodict/py3/tests/test_xmltodict.py b/contrib/python/xmltodict/py3/tests/test_xmltodict.py new file mode 100644 index 00000000000..d778816817f --- /dev/null +++ b/contrib/python/xmltodict/py3/tests/test_xmltodict.py @@ -0,0 +1,382 @@ +from xmltodict import parse, ParsingInterrupted +import unittest + +try: + from io import BytesIO as StringIO +except ImportError: + from xmltodict import StringIO + +from xml.parsers.expat import ParserCreate +from xml.parsers import expat + + +def _encode(s): + try: + return bytes(s, 'ascii') + except (NameError, TypeError): + return s + + +class XMLToDictTestCase(unittest.TestCase): + + def test_string_vs_file(self): + xml = 'data' + self.assertEqual(parse(xml), + parse(StringIO(_encode(xml)))) + + def test_minimal(self): + self.assertEqual(parse(''), + {'a': None}) + self.assertEqual(parse('', force_cdata=True), + {'a': None}) + + def test_simple(self): + self.assertEqual(parse('data'), + {'a': 'data'}) + + def test_force_cdata(self): + self.assertEqual(parse('data', force_cdata=True), + {'a': {'#text': 'data'}}) + + def test_custom_cdata(self): + self.assertEqual(parse('data', + force_cdata=True, + cdata_key='_CDATA_'), + {'a': {'_CDATA_': 'data'}}) + + def test_list(self): + self.assertEqual(parse('123'), + {'a': {'b': ['1', '2', '3']}}) + + def test_attrib(self): + self.assertEqual(parse(''), + {'a': {'@href': 'xyz'}}) + + def test_skip_attrib(self): + self.assertEqual(parse('', xml_attribs=False), + {'a': None}) + + def test_custom_attrib(self): + self.assertEqual(parse('', + attr_prefix='!'), + {'a': {'!href': 'xyz'}}) + + def test_attrib_and_cdata(self): + self.assertEqual(parse('123'), + {'a': {'@href': 'xyz', '#text': '123'}}) + + def test_semi_structured(self): + self.assertEqual(parse('abcdef'), + {'a': {'b': None, '#text': 'abcdef'}}) + self.assertEqual(parse('abcdef', + cdata_separator='\n'), + {'a': {'b': None, '#text': 'abc\ndef'}}) + + def test_nested_semi_structured(self): + self.assertEqual(parse('abc123456def'), + {'a': {'#text': 'abcdef', 'b': { + '#text': '123456', 'c': None}}}) + + def test_skip_whitespace(self): + xml = """ + + + + + + + + + hello + + """ + self.assertEqual( + parse(xml), + {'root': {'emptya': None, + 'emptyb': {'@attr': 'attrvalue'}, + 'value': 'hello'}}) + + def test_keep_whitespace(self): + xml = " " + self.assertEqual(parse(xml), dict(root=None)) + self.assertEqual(parse(xml, strip_whitespace=False), + dict(root=' ')) + + def test_streaming(self): + def cb(path, item): + cb.count += 1 + self.assertEqual(path, [('a', {'x': 'y'}), ('b', None)]) + self.assertEqual(item, str(cb.count)) + return True + cb.count = 0 + parse('123', + item_depth=2, item_callback=cb) + self.assertEqual(cb.count, 3) + + def test_streaming_interrupt(self): + cb = lambda path, item: False + self.assertRaises(ParsingInterrupted, + parse, 'x', + item_depth=1, item_callback=cb) + + def test_postprocessor(self): + def postprocessor(path, key, value): + try: + return key + ':int', int(value) + except (ValueError, TypeError): + return key, value + self.assertEqual({'a': {'b:int': [1, 2], 'b': 'x'}}, + parse('12x', + postprocessor=postprocessor)) + + def test_postprocessor_attribute(self): + def postprocessor(path, key, value): + try: + return key + ':int', int(value) + except (ValueError, TypeError): + return key, value + self.assertEqual({'a': {'@b:int': 1}}, + parse('', + postprocessor=postprocessor)) + + def test_postprocessor_skip(self): + def postprocessor(path, key, value): + if key == 'b': + value = int(value) + if value == 3: + return None + return key, value + self.assertEqual({'a': {'b': [1, 2]}}, + parse('123', + postprocessor=postprocessor)) + + def test_unicode(self): + try: + value = unichr(39321) + except NameError: + value = chr(39321) + self.assertEqual({'a': value}, + parse('%s' % value)) + + def test_encoded_string(self): + try: + value = unichr(39321) + except NameError: + value = chr(39321) + xml = '%s' % value + self.assertEqual(parse(xml), + parse(xml.encode('utf-8'))) + + def test_namespace_support(self): + xml = """ + + 1 + 2 + 3 + + """ + d = { + 'http://defaultns.com/:root': { + 'http://defaultns.com/:x': { + '@xmlns': { + '': 'http://defaultns.com/', + 'a': 'http://a.com/', + 'b': 'http://b.com/', + }, + '@http://a.com/:attr': 'val', + '#text': '1', + }, + 'http://a.com/:y': '2', + 'http://b.com/:z': '3', + } + } + res = parse(xml, process_namespaces=True) + self.assertEqual(res, d) + + def test_namespace_collapse(self): + xml = """ + + 1 + 2 + 3 + + """ + namespaces = { + 'http://defaultns.com/': '', + 'http://a.com/': 'ns_a', + } + d = { + 'root': { + 'x': { + '@xmlns': { + '': 'http://defaultns.com/', + 'a': 'http://a.com/', + 'b': 'http://b.com/', + }, + '@ns_a:attr': 'val', + '#text': '1', + }, + 'ns_a:y': '2', + 'http://b.com/:z': '3', + }, + } + res = parse(xml, process_namespaces=True, namespaces=namespaces) + self.assertEqual(res, d) + + def test_namespace_ignore(self): + xml = """ + + 1 + 2 + 3 + + """ + d = { + 'root': { + '@xmlns': 'http://defaultns.com/', + '@xmlns:a': 'http://a.com/', + '@xmlns:b': 'http://b.com/', + 'x': '1', + 'a:y': '2', + 'b:z': '3', + }, + } + self.assertEqual(parse(xml), d) + + def test_force_list_basic(self): + xml = """ + + + server1 + os1 + + + """ + expectedResult = { + 'servers': { + 'server': [ + { + 'name': 'server1', + 'os': 'os1', + }, + ], + } + } + self.assertEqual(parse(xml, force_list=('server',)), expectedResult) + + def test_force_list_callable(self): + xml = """ + + + + server1 + os1 + + + + + + + """ + + def force_list(path, key, value): + """Only return True for servers/server, but not for skip/server.""" + if key != 'server': + return False + return path and path[-1][0] == 'servers' + + expectedResult = { + 'config': { + 'servers': { + 'server': [ + { + 'name': 'server1', + 'os': 'os1', + }, + ], + }, + 'skip': { + 'server': None, + }, + }, + } + self.assertEqual(parse(xml, force_list=force_list, dict_constructor=dict), expectedResult) + + def test_disable_entities_true_ignores_xmlbomb(self): + xml = """ + + + + ]> + &c; + """ + expectedResult = {'bomb': None} + try: + parse_attempt = parse(xml, disable_entities=True) + except expat.ExpatError: + self.assertTrue(True) + else: + self.assertEqual(parse_attempt, expectedResult) + + def test_disable_entities_false_returns_xmlbomb(self): + xml = """ + + + + ]> + &c; + """ + bomb = "1234567890" * 64 + expectedResult = {'bomb': bomb} + self.assertEqual(parse(xml, disable_entities=False), expectedResult) + + def test_disable_entities_true_ignores_external_dtd(self): + xml = """ + + ]> + ⅇ + """ + expectedResult = {'root': None} + try: + parse_attempt = parse(xml, disable_entities=True) + except expat.ExpatError: + self.assertTrue(True) + else: + self.assertEqual(parse_attempt, expectedResult) + + def test_disable_entities_true_attempts_external_dtd(self): + xml = """ + + ]> + ⅇ + """ + + def raising_external_ref_handler(*args, **kwargs): + parser = ParserCreate(*args, **kwargs) + parser.ExternalEntityRefHandler = lambda *x: 0 + try: + feature = "http://apache.org/xml/features/disallow-doctype-decl" + parser._reader.setFeature(feature, True) + except AttributeError: + pass + return parser + expat.ParserCreate = raising_external_ref_handler + # Using this try/catch because a TypeError is thrown before + # the ExpatError, and Python 2.6 is confused by that. + try: + parse(xml, disable_entities=False, expat=expat) + except expat.ExpatError: + self.assertTrue(True) + else: + self.assertTrue(False) + expat.ParserCreate = ParserCreate diff --git a/contrib/python/xmltodict/py3/xmltodict.py b/contrib/python/xmltodict/py3/xmltodict.py new file mode 100644 index 00000000000..d6dbcd7a700 --- /dev/null +++ b/contrib/python/xmltodict/py3/xmltodict.py @@ -0,0 +1,488 @@ +#!/usr/bin/env python +"Makes working with XML feel like you are working with JSON" + +try: + from defusedexpat import pyexpat as expat +except ImportError: + from xml.parsers import expat +from xml.sax.saxutils import XMLGenerator +from xml.sax.xmlreader import AttributesImpl +try: # pragma no cover + from cStringIO import StringIO +except ImportError: # pragma no cover + try: + from StringIO import StringIO + except ImportError: + from io import StringIO + +from collections import OrderedDict + +try: # pragma no cover + _basestring = basestring +except NameError: # pragma no cover + _basestring = str +try: # pragma no cover + _unicode = unicode +except NameError: # pragma no cover + _unicode = str + +__author__ = 'Martin Blech' +__version__ = '0.12.0' +__license__ = 'MIT' + + +class ParsingInterrupted(Exception): + pass + + +class _DictSAXHandler(object): + def __init__(self, + item_depth=0, + item_callback=lambda *args: True, + xml_attribs=True, + attr_prefix='@', + cdata_key='#text', + force_cdata=False, + cdata_separator='', + postprocessor=None, + dict_constructor=OrderedDict, + strip_whitespace=True, + namespace_separator=':', + namespaces=None, + force_list=None): + self.path = [] + self.stack = [] + self.data = [] + self.item = None + self.item_depth = item_depth + self.xml_attribs = xml_attribs + self.item_callback = item_callback + self.attr_prefix = attr_prefix + self.cdata_key = cdata_key + self.force_cdata = force_cdata + self.cdata_separator = cdata_separator + self.postprocessor = postprocessor + self.dict_constructor = dict_constructor + self.strip_whitespace = strip_whitespace + self.namespace_separator = namespace_separator + self.namespaces = namespaces + self.namespace_declarations = OrderedDict() + self.force_list = force_list + + def _build_name(self, full_name): + if not self.namespaces: + return full_name + i = full_name.rfind(self.namespace_separator) + if i == -1: + return full_name + namespace, name = full_name[:i], full_name[i+1:] + short_namespace = self.namespaces.get(namespace, namespace) + if not short_namespace: + return name + else: + return self.namespace_separator.join((short_namespace, name)) + + def _attrs_to_dict(self, attrs): + if isinstance(attrs, dict): + return attrs + return self.dict_constructor(zip(attrs[0::2], attrs[1::2])) + + def startNamespaceDecl(self, prefix, uri): + self.namespace_declarations[prefix or ''] = uri + + def startElement(self, full_name, attrs): + name = self._build_name(full_name) + attrs = self._attrs_to_dict(attrs) + if attrs and self.namespace_declarations: + attrs['xmlns'] = self.namespace_declarations + self.namespace_declarations = OrderedDict() + self.path.append((name, attrs or None)) + if len(self.path) > self.item_depth: + self.stack.append((self.item, self.data)) + if self.xml_attribs: + attr_entries = [] + for key, value in attrs.items(): + key = self.attr_prefix+self._build_name(key) + if self.postprocessor: + entry = self.postprocessor(self.path, key, value) + else: + entry = (key, value) + if entry: + attr_entries.append(entry) + attrs = self.dict_constructor(attr_entries) + else: + attrs = None + self.item = attrs or None + self.data = [] + + def endElement(self, full_name): + name = self._build_name(full_name) + if len(self.path) == self.item_depth: + item = self.item + if item is None: + item = (None if not self.data + else self.cdata_separator.join(self.data)) + + should_continue = self.item_callback(self.path, item) + if not should_continue: + raise ParsingInterrupted() + if len(self.stack): + data = (None if not self.data + else self.cdata_separator.join(self.data)) + item = self.item + self.item, self.data = self.stack.pop() + if self.strip_whitespace and data: + data = data.strip() or None + if data and self.force_cdata and item is None: + item = self.dict_constructor() + if item is not None: + if data: + self.push_data(item, self.cdata_key, data) + self.item = self.push_data(self.item, name, item) + else: + self.item = self.push_data(self.item, name, data) + else: + self.item = None + self.data = [] + self.path.pop() + + def characters(self, data): + if not self.data: + self.data = [data] + else: + self.data.append(data) + + def push_data(self, item, key, data): + if self.postprocessor is not None: + result = self.postprocessor(self.path, key, data) + if result is None: + return item + key, data = result + if item is None: + item = self.dict_constructor() + try: + value = item[key] + if isinstance(value, list): + value.append(data) + else: + item[key] = [value, data] + except KeyError: + if self._should_force_list(key, data): + item[key] = [data] + else: + item[key] = data + return item + + def _should_force_list(self, key, value): + if not self.force_list: + return False + if isinstance(self.force_list, bool): + return self.force_list + try: + return key in self.force_list + except TypeError: + return self.force_list(self.path[:-1], key, value) + + +def parse(xml_input, encoding=None, expat=expat, process_namespaces=False, + namespace_separator=':', disable_entities=True, **kwargs): + """Parse the given XML input and convert it into a dictionary. + + `xml_input` can either be a `string` or a file-like object. + + If `xml_attribs` is `True`, element attributes are put in the dictionary + among regular child elements, using `@` as a prefix to avoid collisions. If + set to `False`, they are just ignored. + + Simple example:: + + >>> import xmltodict + >>> doc = xmltodict.parse(\"\"\" + ... + ... 1 + ... 2 + ... + ... \"\"\") + >>> doc['a']['@prop'] + u'x' + >>> doc['a']['b'] + [u'1', u'2'] + + If `item_depth` is `0`, the function returns a dictionary for the root + element (default behavior). Otherwise, it calls `item_callback` every time + an item at the specified depth is found and returns `None` in the end + (streaming mode). + + The callback function receives two parameters: the `path` from the document + root to the item (name-attribs pairs), and the `item` (dict). If the + callback's return value is false-ish, parsing will be stopped with the + :class:`ParsingInterrupted` exception. + + Streaming example:: + + >>> def handle(path, item): + ... print('path:%s item:%s' % (path, item)) + ... return True + ... + >>> xmltodict.parse(\"\"\" + ... + ... 1 + ... 2 + ... \"\"\", item_depth=2, item_callback=handle) + path:[(u'a', {u'prop': u'x'}), (u'b', None)] item:1 + path:[(u'a', {u'prop': u'x'}), (u'b', None)] item:2 + + The optional argument `postprocessor` is a function that takes `path`, + `key` and `value` as positional arguments and returns a new `(key, value)` + pair where both `key` and `value` may have changed. Usage example:: + + >>> def postprocessor(path, key, value): + ... try: + ... return key + ':int', int(value) + ... except (ValueError, TypeError): + ... return key, value + >>> xmltodict.parse('12x', + ... postprocessor=postprocessor) + OrderedDict([(u'a', OrderedDict([(u'b:int', [1, 2]), (u'b', u'x')]))]) + + You can pass an alternate version of `expat` (such as `defusedexpat`) by + using the `expat` parameter. E.g: + + >>> import defusedexpat + >>> xmltodict.parse('hello', expat=defusedexpat.pyexpat) + OrderedDict([(u'a', u'hello')]) + + You can use the force_list argument to force lists to be created even + when there is only a single child of a given level of hierarchy. The + force_list argument is a tuple of keys. If the key for a given level + of hierarchy is in the force_list argument, that level of hierarchy + will have a list as a child (even if there is only one sub-element). + The index_keys operation takes precendence over this. This is applied + after any user-supplied postprocessor has already run. + + For example, given this input: + + + host1 + Linux + + + em0 + 10.0.0.1 + + + + + + If called with force_list=('interface',), it will produce + this dictionary: + {'servers': + {'server': + {'name': 'host1', + 'os': 'Linux'}, + 'interfaces': + {'interface': + [ {'name': 'em0', 'ip_address': '10.0.0.1' } ] } } } + + `force_list` can also be a callable that receives `path`, `key` and + `value`. This is helpful in cases where the logic that decides whether + a list should be forced is more complex. + """ + handler = _DictSAXHandler(namespace_separator=namespace_separator, + **kwargs) + if isinstance(xml_input, _unicode): + if not encoding: + encoding = 'utf-8' + xml_input = xml_input.encode(encoding) + if not process_namespaces: + namespace_separator = None + parser = expat.ParserCreate( + encoding, + namespace_separator + ) + try: + parser.ordered_attributes = True + except AttributeError: + # Jython's expat does not support ordered_attributes + pass + parser.StartNamespaceDeclHandler = handler.startNamespaceDecl + parser.StartElementHandler = handler.startElement + parser.EndElementHandler = handler.endElement + parser.CharacterDataHandler = handler.characters + parser.buffer_text = True + if disable_entities: + try: + # Attempt to disable DTD in Jython's expat parser (Xerces-J). + feature = "http://apache.org/xml/features/disallow-doctype-decl" + parser._reader.setFeature(feature, True) + except AttributeError: + # For CPython / expat parser. + # Anything not handled ends up here and entities aren't expanded. + parser.DefaultHandler = lambda x: None + # Expects an integer return; zero means failure -> expat.ExpatError. + parser.ExternalEntityRefHandler = lambda *x: 1 + if hasattr(xml_input, 'read'): + parser.ParseFile(xml_input) + else: + parser.Parse(xml_input, True) + return handler.item + + +def _process_namespace(name, namespaces, ns_sep=':', attr_prefix='@'): + if not namespaces: + return name + try: + ns, name = name.rsplit(ns_sep, 1) + except ValueError: + pass + else: + ns_res = namespaces.get(ns.strip(attr_prefix)) + name = '{}{}{}{}'.format( + attr_prefix if ns.startswith(attr_prefix) else '', + ns_res, ns_sep, name) if ns_res else name + return name + + +def _emit(key, value, content_handler, + attr_prefix='@', + cdata_key='#text', + depth=0, + preprocessor=None, + pretty=False, + newl='\n', + indent='\t', + namespace_separator=':', + namespaces=None, + full_document=True): + key = _process_namespace(key, namespaces, namespace_separator, attr_prefix) + if preprocessor is not None: + result = preprocessor(key, value) + if result is None: + return + key, value = result + if (not hasattr(value, '__iter__') + or isinstance(value, _basestring) + or isinstance(value, dict)): + value = [value] + for index, v in enumerate(value): + if full_document and depth == 0 and index > 0: + raise ValueError('document with multiple roots') + if v is None: + v = OrderedDict() + elif isinstance(v, bool): + if v: + v = _unicode('true') + else: + v = _unicode('false') + elif not isinstance(v, dict): + v = _unicode(v) + if isinstance(v, _basestring): + v = OrderedDict(((cdata_key, v),)) + cdata = None + attrs = OrderedDict() + children = [] + for ik, iv in v.items(): + if ik == cdata_key: + cdata = iv + continue + if ik.startswith(attr_prefix): + ik = _process_namespace(ik, namespaces, namespace_separator, + attr_prefix) + if ik == '@xmlns' and isinstance(iv, dict): + for k, v in iv.items(): + attr = 'xmlns{}'.format(':{}'.format(k) if k else '') + attrs[attr] = _unicode(v) + continue + if not isinstance(iv, _unicode): + iv = _unicode(iv) + attrs[ik[len(attr_prefix):]] = iv + continue + children.append((ik, iv)) + if pretty: + content_handler.ignorableWhitespace(depth * indent) + content_handler.startElement(key, AttributesImpl(attrs)) + if pretty and children: + content_handler.ignorableWhitespace(newl) + for child_key, child_value in children: + _emit(child_key, child_value, content_handler, + attr_prefix, cdata_key, depth+1, preprocessor, + pretty, newl, indent, namespaces=namespaces, + namespace_separator=namespace_separator) + if cdata is not None: + content_handler.characters(cdata) + if pretty and children: + content_handler.ignorableWhitespace(depth * indent) + content_handler.endElement(key) + if pretty and depth: + content_handler.ignorableWhitespace(newl) + + +def unparse(input_dict, output=None, encoding='utf-8', full_document=True, + short_empty_elements=False, + **kwargs): + """Emit an XML document for the given `input_dict` (reverse of `parse`). + + The resulting XML document is returned as a string, but if `output` (a + file-like object) is specified, it is written there instead. + + Dictionary keys prefixed with `attr_prefix` (default=`'@'`) are interpreted + as XML node attributes, whereas keys equal to `cdata_key` + (default=`'#text'`) are treated as character data. + + The `pretty` parameter (default=`False`) enables pretty-printing. In this + mode, lines are terminated with `'\n'` and indented with `'\t'`, but this + can be customized with the `newl` and `indent` parameters. + + """ + if full_document and len(input_dict) != 1: + raise ValueError('Document must have exactly one root.') + must_return = False + if output is None: + output = StringIO() + must_return = True + if short_empty_elements: + content_handler = XMLGenerator(output, encoding, True) + else: + content_handler = XMLGenerator(output, encoding) + if full_document: + content_handler.startDocument() + for key, value in input_dict.items(): + _emit(key, value, content_handler, full_document=full_document, + **kwargs) + if full_document: + content_handler.endDocument() + if must_return: + value = output.getvalue() + try: # pragma no cover + value = value.decode(encoding) + except AttributeError: # pragma no cover + pass + return value + + +if __name__ == '__main__': # pragma: no cover + import sys + import marshal + try: + stdin = sys.stdin.buffer + stdout = sys.stdout.buffer + except AttributeError: + stdin = sys.stdin + stdout = sys.stdout + + (item_depth,) = sys.argv[1:] + item_depth = int(item_depth) + + def handle_item(path, item): + marshal.dump((path, item), stdout) + return True + + try: + root = parse(stdin, + item_depth=item_depth, + item_callback=handle_item, + dict_constructor=dict) + if item_depth == 0: + handle_item([], root) + except KeyboardInterrupt: + pass diff --git a/contrib/python/xmltodict/tests/test_dicttoxml.py b/contrib/python/xmltodict/tests/test_dicttoxml.py deleted file mode 100644 index ebe8a5b731d..00000000000 --- a/contrib/python/xmltodict/tests/test_dicttoxml.py +++ /dev/null @@ -1,207 +0,0 @@ -import sys -from xmltodict import parse, unparse -from collections import OrderedDict - -import unittest -import re -from textwrap import dedent - -IS_JYTHON = sys.platform.startswith('java') - -_HEADER_RE = re.compile(r'^[^\n]*\n') - - -def _strip(fullxml): - return _HEADER_RE.sub('', fullxml) - - -class DictToXMLTestCase(unittest.TestCase): - def test_root(self): - obj = {'a': None} - self.assertEqual(obj, parse(unparse(obj))) - self.assertEqual(unparse(obj), unparse(parse(unparse(obj)))) - - def test_simple_cdata(self): - obj = {'a': 'b'} - self.assertEqual(obj, parse(unparse(obj))) - self.assertEqual(unparse(obj), unparse(parse(unparse(obj)))) - - def test_cdata(self): - obj = {'a': {'#text': 'y'}} - self.assertEqual(obj, parse(unparse(obj), force_cdata=True)) - self.assertEqual(unparse(obj), unparse(parse(unparse(obj)))) - - def test_attrib(self): - obj = {'a': {'@href': 'x'}} - self.assertEqual(obj, parse(unparse(obj))) - self.assertEqual(unparse(obj), unparse(parse(unparse(obj)))) - - def test_attrib_and_cdata(self): - obj = {'a': {'@href': 'x', '#text': 'y'}} - self.assertEqual(obj, parse(unparse(obj))) - self.assertEqual(unparse(obj), unparse(parse(unparse(obj)))) - - def test_list(self): - obj = {'a': {'b': ['1', '2', '3']}} - self.assertEqual(obj, parse(unparse(obj))) - self.assertEqual(unparse(obj), unparse(parse(unparse(obj)))) - - def test_generator(self): - obj = {'a': {'b': ['1', '2', '3']}} - - def lazy_obj(): - return {'a': {'b': (i for i in ('1', '2', '3'))}} - self.assertEqual(obj, parse(unparse(lazy_obj()))) - self.assertEqual(unparse(lazy_obj()), - unparse(parse(unparse(lazy_obj())))) - - def test_no_root(self): - self.assertRaises(ValueError, unparse, {}) - - def test_multiple_roots(self): - self.assertRaises(ValueError, unparse, {'a': '1', 'b': '2'}) - self.assertRaises(ValueError, unparse, {'a': ['1', '2', '3']}) - - def test_no_root_nofulldoc(self): - self.assertEqual(unparse({}, full_document=False), '') - - def test_multiple_roots_nofulldoc(self): - obj = OrderedDict((('a', 1), ('b', 2))) - xml = unparse(obj, full_document=False) - self.assertEqual(xml, '12') - obj = {'a': [1, 2]} - xml = unparse(obj, full_document=False) - self.assertEqual(xml, '12') - - def test_nested(self): - obj = {'a': {'b': '1', 'c': '2'}} - self.assertEqual(obj, parse(unparse(obj))) - self.assertEqual(unparse(obj), unparse(parse(unparse(obj)))) - obj = {'a': {'b': {'c': {'@a': 'x', '#text': 'y'}}}} - self.assertEqual(obj, parse(unparse(obj))) - self.assertEqual(unparse(obj), unparse(parse(unparse(obj)))) - - def test_semistructured(self): - xml = 'abcefg' - self.assertEqual(_strip(unparse(parse(xml))), - 'abcefg') - - def test_preprocessor(self): - obj = {'a': OrderedDict((('b:int', [1, 2]), ('b', 'c')))} - - def p(key, value): - try: - key, _ = key.split(':') - except ValueError: - pass - return key, value - - self.assertEqual(_strip(unparse(obj, preprocessor=p)), - '12c') - - def test_preprocessor_skipkey(self): - obj = {'a': {'b': 1, 'c': 2}} - - def p(key, value): - if key == 'b': - return None - return key, value - - self.assertEqual(_strip(unparse(obj, preprocessor=p)), - '2') - - if not IS_JYTHON: - # Jython's SAX does not preserve attribute order - def test_attr_order_roundtrip(self): - xml = '' - self.assertEqual(xml, _strip(unparse(parse(xml)))) - - def test_pretty_print(self): - obj = {'a': OrderedDict(( - ('b', [{'c': [1, 2]}, 3]), - ('x', 'y'), - ))} - newl = '\n' - indent = '....' - xml = dedent('''\ - - - .... - ........1 - ........2 - .... - ....3 - ....y - ''') - self.assertEqual(xml, unparse(obj, pretty=True, - newl=newl, indent=indent)) - - def test_encoding(self): - try: - value = unichr(39321) - except NameError: - value = chr(39321) - obj = {'a': value} - utf8doc = unparse(obj, encoding='utf-8') - latin1doc = unparse(obj, encoding='iso-8859-1') - self.assertEqual(parse(utf8doc), parse(latin1doc)) - self.assertEqual(parse(utf8doc), obj) - - def test_fulldoc(self): - xml_declaration_re = re.compile( - '^' + re.escape('')) - self.assertTrue(xml_declaration_re.match(unparse({'a': 1}))) - self.assertFalse( - xml_declaration_re.match(unparse({'a': 1}, full_document=False))) - - def test_non_string_value(self): - obj = {'a': 1} - self.assertEqual('1', _strip(unparse(obj))) - - def test_non_string_attr(self): - obj = {'a': {'@attr': 1}} - self.assertEqual('', _strip(unparse(obj))) - - def test_short_empty_elements(self): - if sys.version_info[0] < 3: - return - obj = {'a': None} - self.assertEqual('', _strip(unparse(obj, short_empty_elements=True))) - - def test_namespace_support(self): - obj = OrderedDict(( - ('http://defaultns.com/|root', OrderedDict(( - ('@xmlns', OrderedDict(( - ('', 'http://defaultns.com/'), - ('a', 'http://a.com/'), - ('b', 'http://b.com/'), - ))), - ('http://defaultns.com/|x', OrderedDict(( - ('@http://a.com/|attr', 'val'), - ('#text', '1'), - ))), - ('http://a.com/|y', '2'), - ('http://b.com/|z', '3'), - ))), - )) - ns = { - 'http://defaultns.com/': '', - 'http://a.com/': 'a', - 'http://b.com/': 'b', - } - - expected_xml = ''' -123''' - xml = unparse(obj, namespaces=ns) - - self.assertEqual(xml, expected_xml) - - def test_boolean_unparse(self): - expected_xml = '\ntrue' - xml = unparse(dict(x=True)) - self.assertEqual(xml, expected_xml) - - expected_xml = '\nfalse' - xml = unparse(dict(x=False)) - self.assertEqual(xml, expected_xml) diff --git a/contrib/python/xmltodict/tests/test_xmltodict.py b/contrib/python/xmltodict/tests/test_xmltodict.py deleted file mode 100644 index 1d01bb8e269..00000000000 --- a/contrib/python/xmltodict/tests/test_xmltodict.py +++ /dev/null @@ -1,382 +0,0 @@ -from xmltodict import parse, ParsingInterrupted -import unittest - -try: - from io import BytesIO as StringIO -except ImportError: - from xmltodict import StringIO - -from xml.parsers.expat import ParserCreate -from xml.parsers import expat - - -def _encode(s): - try: - return bytes(s, 'ascii') - except (NameError, TypeError): - return s - - -class XMLToDictTestCase(unittest.TestCase): - - def test_string_vs_file(self): - xml = 'data' - self.assertEqual(parse(xml), - parse(StringIO(_encode(xml)))) - - def test_minimal(self): - self.assertEqual(parse(''), - {'a': None}) - self.assertEqual(parse('', force_cdata=True), - {'a': None}) - - def test_simple(self): - self.assertEqual(parse('data'), - {'a': 'data'}) - - def test_force_cdata(self): - self.assertEqual(parse('data', force_cdata=True), - {'a': {'#text': 'data'}}) - - def test_custom_cdata(self): - self.assertEqual(parse('data', - force_cdata=True, - cdata_key='_CDATA_'), - {'a': {'_CDATA_': 'data'}}) - - def test_list(self): - self.assertEqual(parse('123'), - {'a': {'b': ['1', '2', '3']}}) - - def test_attrib(self): - self.assertEqual(parse(''), - {'a': {'@href': 'xyz'}}) - - def test_skip_attrib(self): - self.assertEqual(parse('', xml_attribs=False), - {'a': None}) - - def test_custom_attrib(self): - self.assertEqual(parse('', - attr_prefix='!'), - {'a': {'!href': 'xyz'}}) - - def test_attrib_and_cdata(self): - self.assertEqual(parse('123'), - {'a': {'@href': 'xyz', '#text': '123'}}) - - def test_semi_structured(self): - self.assertEqual(parse('abcdef'), - {'a': {'b': None, '#text': 'abcdef'}}) - self.assertEqual(parse('abcdef', - cdata_separator='\n'), - {'a': {'b': None, '#text': 'abc\ndef'}}) - - def test_nested_semi_structured(self): - self.assertEqual(parse('abc123456def'), - {'a': {'#text': 'abcdef', 'b': { - '#text': '123456', 'c': None}}}) - - def test_skip_whitespace(self): - xml = """ - - - - - - - - - hello - - """ - self.assertEqual( - parse(xml), - {'root': {'emptya': None, - 'emptyb': {'@attr': 'attrvalue'}, - 'value': 'hello'}}) - - def test_keep_whitespace(self): - xml = " " - self.assertEqual(parse(xml), dict(root=None)) - self.assertEqual(parse(xml, strip_whitespace=False), - dict(root=' ')) - - def test_streaming(self): - def cb(path, item): - cb.count += 1 - self.assertEqual(path, [('a', {'x': 'y'}), ('b', None)]) - self.assertEqual(item, str(cb.count)) - return True - cb.count = 0 - parse('123', - item_depth=2, item_callback=cb) - self.assertEqual(cb.count, 3) - - def test_streaming_interrupt(self): - cb = lambda path, item: False - self.assertRaises(ParsingInterrupted, - parse, 'x', - item_depth=1, item_callback=cb) - - def test_postprocessor(self): - def postprocessor(path, key, value): - try: - return key + ':int', int(value) - except (ValueError, TypeError): - return key, value - self.assertEqual({'a': {'b:int': [1, 2], 'b': 'x'}}, - parse('12x', - postprocessor=postprocessor)) - - def test_postprocessor_attribute(self): - def postprocessor(path, key, value): - try: - return key + ':int', int(value) - except (ValueError, TypeError): - return key, value - self.assertEqual({'a': {'@b:int': 1}}, - parse('', - postprocessor=postprocessor)) - - def test_postprocessor_skip(self): - def postprocessor(path, key, value): - if key == 'b': - value = int(value) - if value == 3: - return None - return key, value - self.assertEqual({'a': {'b': [1, 2]}}, - parse('123', - postprocessor=postprocessor)) - - def test_unicode(self): - try: - value = unichr(39321) - except NameError: - value = chr(39321) - self.assertEqual({'a': value}, - parse('%s' % value)) - - def test_encoded_string(self): - try: - value = unichr(39321) - except NameError: - value = chr(39321) - xml = '%s' % value - self.assertEqual(parse(xml), - parse(xml.encode('utf-8'))) - - def test_namespace_support(self): - xml = """ - - 1 - 2 - 3 - - """ - d = { - 'http://defaultns.com/|root': { - 'http://defaultns.com/|x': { - '@xmlns': { - '': 'http://defaultns.com/', - 'a': 'http://a.com/', - 'b': 'http://b.com/', - }, - '@http://a.com/|attr': 'val', - '#text': '1', - }, - 'http://a.com/|y': '2', - 'http://b.com/|z': '3', - } - } - res = parse(xml, process_namespaces=True) - self.assertEqual(res, d) - - def test_namespace_collapse(self): - xml = """ - - 1 - 2 - 3 - - """ - namespaces = { - 'http://defaultns.com/': '', - 'http://a.com/': 'ns_a', - } - d = { - 'root': { - 'x': { - '@xmlns': { - '': 'http://defaultns.com/', - 'a': 'http://a.com/', - 'b': 'http://b.com/', - }, - '@ns_a|attr': 'val', - '#text': '1', - }, - 'ns_a|y': '2', - 'http://b.com/|z': '3', - }, - } - res = parse(xml, process_namespaces=True, namespaces=namespaces) - self.assertEqual(res, d) - - def test_namespace_ignore(self): - xml = """ - - 1 - 2 - 3 - - """ - d = { - 'root': { - '@xmlns': 'http://defaultns.com/', - '@xmlns:a': 'http://a.com/', - '@xmlns:b': 'http://b.com/', - 'x': '1', - 'a:y': '2', - 'b:z': '3', - }, - } - self.assertEqual(parse(xml), d) - - def test_force_list_basic(self): - xml = """ - - - server1 - os1 - - - """ - expectedResult = { - 'servers': { - 'server': [ - { - 'name': 'server1', - 'os': 'os1', - }, - ], - } - } - self.assertEqual(parse(xml, force_list=('server',)), expectedResult) - - def test_force_list_callable(self): - xml = """ - - - - server1 - os1 - - - - - - - """ - - def force_list(path, key, value): - """Only return True for servers/server, but not for skip/server.""" - if key != 'server': - return False - return path and path[-1][0] == 'servers' - - expectedResult = { - 'config': { - 'servers': { - 'server': [ - { - 'name': 'server1', - 'os': 'os1', - }, - ], - }, - 'skip': { - 'server': None, - }, - }, - } - self.assertEqual(parse(xml, force_list=force_list, dict_constructor=dict), expectedResult) - - def test_disable_entities_true_ignores_xmlbomb(self): - xml = """ - - - - ]> - &c; - """ - expectedResult = {'bomb': None} - try: - parse_attempt = parse(xml, disable_entities=True) - except expat.ExpatError: - self.assertTrue(True) - else: - self.assertEqual(parse_attempt, expectedResult) - - def test_disable_entities_false_returns_xmlbomb(self): - xml = """ - - - - ]> - &c; - """ - bomb = "1234567890" * 64 - expectedResult = {'bomb': bomb} - self.assertEqual(parse(xml, disable_entities=False), expectedResult) - - def test_disable_entities_true_ignores_external_dtd(self): - xml = """ - - ]> - ⅇ - """ - expectedResult = {'root': None} - try: - parse_attempt = parse(xml, disable_entities=True) - except expat.ExpatError: - self.assertTrue(True) - else: - self.assertEqual(parse_attempt, expectedResult) - - def test_disable_entities_true_attempts_external_dtd(self): - xml = """ - - ]> - ⅇ - """ - - def raising_external_ref_handler(*args, **kwargs): - parser = ParserCreate(*args, **kwargs) - parser.ExternalEntityRefHandler = lambda *x: 0 - try: - feature = "http://apache.org/xml/features/disallow-doctype-decl" - parser._reader.setFeature(feature, True) - except AttributeError: - pass - return parser - expat.ParserCreate = raising_external_ref_handler - # Using this try/catch because a TypeError is thrown before - # the ExpatError, and Python 2.6 is confused by that. - try: - parse(xml, disable_entities=False, expat=expat) - except expat.ExpatError: - self.assertTrue(True) - else: - self.assertTrue(False) - expat.ParserCreate = ParserCreate diff --git a/contrib/python/xmltodict/xmltodict.py b/contrib/python/xmltodict/xmltodict.py deleted file mode 100644 index 5027283c3e0..00000000000 --- a/contrib/python/xmltodict/xmltodict.py +++ /dev/null @@ -1,488 +0,0 @@ -#!/usr/bin/env python -"Makes working with XML feel like you are working with JSON" - -try: - from defusedexpat import pyexpat as expat -except ImportError: - from xml.parsers import expat -from xml.sax.saxutils import XMLGenerator -from xml.sax.xmlreader import AttributesImpl -try: # pragma no cover - from cStringIO import StringIO -except ImportError: # pragma no cover - try: - from StringIO import StringIO - except ImportError: - from io import StringIO - -from collections import OrderedDict - -try: # pragma no cover - _basestring = basestring -except NameError: # pragma no cover - _basestring = str -try: # pragma no cover - _unicode = unicode -except NameError: # pragma no cover - _unicode = str - -__author__ = 'Martin Blech' -__version__ = '0.12.0' -__license__ = 'MIT' - - -class ParsingInterrupted(Exception): - pass - - -class _DictSAXHandler(object): - def __init__(self, - item_depth=0, - item_callback=lambda *args: True, - xml_attribs=True, - attr_prefix='@', - cdata_key='#text', - force_cdata=False, - cdata_separator='', - postprocessor=None, - dict_constructor=OrderedDict, - strip_whitespace=True, - namespace_separator='|', - namespaces=None, - force_list=None): - self.path = [] - self.stack = [] - self.data = [] - self.item = None - self.item_depth = item_depth - self.xml_attribs = xml_attribs - self.item_callback = item_callback - self.attr_prefix = attr_prefix - self.cdata_key = cdata_key - self.force_cdata = force_cdata - self.cdata_separator = cdata_separator - self.postprocessor = postprocessor - self.dict_constructor = dict_constructor - self.strip_whitespace = strip_whitespace - self.namespace_separator = namespace_separator - self.namespaces = namespaces - self.namespace_declarations = OrderedDict() - self.force_list = force_list - - def _build_name(self, full_name): - if not self.namespaces: - return full_name - i = full_name.rfind(self.namespace_separator) - if i == -1: - return full_name - namespace, name = full_name[:i], full_name[i+1:] - short_namespace = self.namespaces.get(namespace, namespace) - if not short_namespace: - return name - else: - return self.namespace_separator.join((short_namespace, name)) - - def _attrs_to_dict(self, attrs): - if isinstance(attrs, dict): - return attrs - return self.dict_constructor(zip(attrs[0::2], attrs[1::2])) - - def startNamespaceDecl(self, prefix, uri): - self.namespace_declarations[prefix or ''] = uri - - def startElement(self, full_name, attrs): - name = self._build_name(full_name) - attrs = self._attrs_to_dict(attrs) - if attrs and self.namespace_declarations: - attrs['xmlns'] = self.namespace_declarations - self.namespace_declarations = OrderedDict() - self.path.append((name, attrs or None)) - if len(self.path) > self.item_depth: - self.stack.append((self.item, self.data)) - if self.xml_attribs: - attr_entries = [] - for key, value in attrs.items(): - key = self.attr_prefix+self._build_name(key) - if self.postprocessor: - entry = self.postprocessor(self.path, key, value) - else: - entry = (key, value) - if entry: - attr_entries.append(entry) - attrs = self.dict_constructor(attr_entries) - else: - attrs = None - self.item = attrs or None - self.data = [] - - def endElement(self, full_name): - name = self._build_name(full_name) - if len(self.path) == self.item_depth: - item = self.item - if item is None: - item = (None if not self.data - else self.cdata_separator.join(self.data)) - - should_continue = self.item_callback(self.path, item) - if not should_continue: - raise ParsingInterrupted() - if len(self.stack): - data = (None if not self.data - else self.cdata_separator.join(self.data)) - item = self.item - self.item, self.data = self.stack.pop() - if self.strip_whitespace and data: - data = data.strip() or None - if data and self.force_cdata and item is None: - item = self.dict_constructor() - if item is not None: - if data: - self.push_data(item, self.cdata_key, data) - self.item = self.push_data(self.item, name, item) - else: - self.item = self.push_data(self.item, name, data) - else: - self.item = None - self.data = [] - self.path.pop() - - def characters(self, data): - if not self.data: - self.data = [data] - else: - self.data.append(data) - - def push_data(self, item, key, data): - if self.postprocessor is not None: - result = self.postprocessor(self.path, key, data) - if result is None: - return item - key, data = result - if item is None: - item = self.dict_constructor() - try: - value = item[key] - if isinstance(value, list): - value.append(data) - else: - item[key] = [value, data] - except KeyError: - if self._should_force_list(key, data): - item[key] = [data] - else: - item[key] = data - return item - - def _should_force_list(self, key, value): - if not self.force_list: - return False - if isinstance(self.force_list, bool): - return self.force_list - try: - return key in self.force_list - except TypeError: - return self.force_list(self.path[:-1], key, value) - - -def parse(xml_input, encoding=None, expat=expat, process_namespaces=False, - namespace_separator='|', disable_entities=True, **kwargs): - """Parse the given XML input and convert it into a dictionary. - - `xml_input` can either be a `string` or a file-like object. - - If `xml_attribs` is `True`, element attributes are put in the dictionary - among regular child elements, using `@` as a prefix to avoid collisions. If - set to `False`, they are just ignored. - - Simple example:: - - >>> import xmltodict - >>> doc = xmltodict.parse(\"\"\" - ... - ... 1 - ... 2 - ... - ... \"\"\") - >>> doc['a']['@prop'] - u'x' - >>> doc['a']['b'] - [u'1', u'2'] - - If `item_depth` is `0`, the function returns a dictionary for the root - element (default behavior). Otherwise, it calls `item_callback` every time - an item at the specified depth is found and returns `None` in the end - (streaming mode). - - The callback function receives two parameters: the `path` from the document - root to the item (name-attribs pairs), and the `item` (dict). If the - callback's return value is false-ish, parsing will be stopped with the - :class:`ParsingInterrupted` exception. - - Streaming example:: - - >>> def handle(path, item): - ... print('path:%s item:%s' % (path, item)) - ... return True - ... - >>> xmltodict.parse(\"\"\" - ... - ... 1 - ... 2 - ... \"\"\", item_depth=2, item_callback=handle) - path:[(u'a', {u'prop': u'x'}), (u'b', None)] item:1 - path:[(u'a', {u'prop': u'x'}), (u'b', None)] item:2 - - The optional argument `postprocessor` is a function that takes `path`, - `key` and `value` as positional arguments and returns a new `(key, value)` - pair where both `key` and `value` may have changed. Usage example:: - - >>> def postprocessor(path, key, value): - ... try: - ... return key + ':int', int(value) - ... except (ValueError, TypeError): - ... return key, value - >>> xmltodict.parse('12x', - ... postprocessor=postprocessor) - OrderedDict([(u'a', OrderedDict([(u'b:int', [1, 2]), (u'b', u'x')]))]) - - You can pass an alternate version of `expat` (such as `defusedexpat`) by - using the `expat` parameter. E.g: - - >>> import defusedexpat - >>> xmltodict.parse('hello', expat=defusedexpat.pyexpat) - OrderedDict([(u'a', u'hello')]) - - You can use the force_list argument to force lists to be created even - when there is only a single child of a given level of hierarchy. The - force_list argument is a tuple of keys. If the key for a given level - of hierarchy is in the force_list argument, that level of hierarchy - will have a list as a child (even if there is only one sub-element). - The index_keys operation takes precendence over this. This is applied - after any user-supplied postprocessor has already run. - - For example, given this input: - - - host1 - Linux - - - em0 - 10.0.0.1 - - - - - - If called with force_list=('interface',), it will produce - this dictionary: - {'servers': - {'server': - {'name': 'host1', - 'os': 'Linux'}, - 'interfaces': - {'interface': - [ {'name': 'em0', 'ip_address': '10.0.0.1' } ] } } } - - `force_list` can also be a callable that receives `path`, `key` and - `value`. This is helpful in cases where the logic that decides whether - a list should be forced is more complex. - """ - handler = _DictSAXHandler(namespace_separator=namespace_separator, - **kwargs) - if isinstance(xml_input, _unicode): - if not encoding: - encoding = 'utf-8' - xml_input = xml_input.encode(encoding) - if not process_namespaces: - namespace_separator = None - parser = expat.ParserCreate( - encoding, - namespace_separator - ) - try: - parser.ordered_attributes = True - except AttributeError: - # Jython's expat does not support ordered_attributes - pass - parser.StartNamespaceDeclHandler = handler.startNamespaceDecl - parser.StartElementHandler = handler.startElement - parser.EndElementHandler = handler.endElement - parser.CharacterDataHandler = handler.characters - parser.buffer_text = True - if disable_entities: - try: - # Attempt to disable DTD in Jython's expat parser (Xerces-J). - feature = "http://apache.org/xml/features/disallow-doctype-decl" - parser._reader.setFeature(feature, True) - except AttributeError: - # For CPython / expat parser. - # Anything not handled ends up here and entities aren't expanded. - parser.DefaultHandler = lambda x: None - # Expects an integer return; zero means failure -> expat.ExpatError. - parser.ExternalEntityRefHandler = lambda *x: 1 - if hasattr(xml_input, 'read'): - parser.ParseFile(xml_input) - else: - parser.Parse(xml_input, True) - return handler.item - - -def _process_namespace(name, namespaces, ns_sep='|', attr_prefix='@'): - if not namespaces: - return name - try: - ns, name = name.rsplit(ns_sep, 1) - except ValueError: - pass - else: - ns_res = namespaces.get(ns.strip(attr_prefix)) - name = '{}{}{}{}'.format( - attr_prefix if ns.startswith(attr_prefix) else '', - ns_res, ':', name) if ns_res else name - return name - - -def _emit(key, value, content_handler, - attr_prefix='@', - cdata_key='#text', - depth=0, - preprocessor=None, - pretty=False, - newl='\n', - indent='\t', - namespace_separator='|', - namespaces=None, - full_document=True): - key = _process_namespace(key, namespaces, namespace_separator, attr_prefix) - if preprocessor is not None: - result = preprocessor(key, value) - if result is None: - return - key, value = result - if (not hasattr(value, '__iter__') - or isinstance(value, _basestring) - or isinstance(value, dict)): - value = [value] - for index, v in enumerate(value): - if full_document and depth == 0 and index > 0: - raise ValueError('document with multiple roots') - if v is None: - v = OrderedDict() - elif isinstance(v, bool): - if v: - v = _unicode('true') - else: - v = _unicode('false') - elif not isinstance(v, dict): - v = _unicode(v) - if isinstance(v, _basestring): - v = OrderedDict(((cdata_key, v),)) - cdata = None - attrs = OrderedDict() - children = [] - for ik, iv in v.items(): - if ik == cdata_key: - cdata = iv - continue - if ik.startswith(attr_prefix): - ik = _process_namespace(ik, namespaces, namespace_separator, - attr_prefix) - if ik == '@xmlns' and isinstance(iv, dict): - for k, v in iv.items(): - attr = 'xmlns{}'.format(':{}'.format(k) if k else '') - attrs[attr] = _unicode(v) - continue - if not isinstance(iv, _unicode): - iv = _unicode(iv) - attrs[ik[len(attr_prefix):]] = iv - continue - children.append((ik, iv)) - if pretty: - content_handler.ignorableWhitespace(depth * indent) - content_handler.startElement(key, AttributesImpl(attrs)) - if pretty and children: - content_handler.ignorableWhitespace(newl) - for child_key, child_value in children: - _emit(child_key, child_value, content_handler, - attr_prefix, cdata_key, depth+1, preprocessor, - pretty, newl, indent, namespaces=namespaces, - namespace_separator=namespace_separator) - if cdata is not None: - content_handler.characters(cdata) - if pretty and children: - content_handler.ignorableWhitespace(depth * indent) - content_handler.endElement(key) - if pretty and depth: - content_handler.ignorableWhitespace(newl) - - -def unparse(input_dict, output=None, encoding='utf-8', full_document=True, - short_empty_elements=False, - **kwargs): - """Emit an XML document for the given `input_dict` (reverse of `parse`). - - The resulting XML document is returned as a string, but if `output` (a - file-like object) is specified, it is written there instead. - - Dictionary keys prefixed with `attr_prefix` (default=`'@'`) are interpreted - as XML node attributes, whereas keys equal to `cdata_key` - (default=`'#text'`) are treated as character data. - - The `pretty` parameter (default=`False`) enables pretty-printing. In this - mode, lines are terminated with `'\n'` and indented with `'\t'`, but this - can be customized with the `newl` and `indent` parameters. - - """ - if full_document and len(input_dict) != 1: - raise ValueError('Document must have exactly one root.') - must_return = False - if output is None: - output = StringIO() - must_return = True - if short_empty_elements: - content_handler = XMLGenerator(output, encoding, True) - else: - content_handler = XMLGenerator(output, encoding) - if full_document: - content_handler.startDocument() - for key, value in input_dict.items(): - _emit(key, value, content_handler, full_document=full_document, - **kwargs) - if full_document: - content_handler.endDocument() - if must_return: - value = output.getvalue() - try: # pragma no cover - value = value.decode(encoding) - except AttributeError: # pragma no cover - pass - return value - - -if __name__ == '__main__': # pragma: no cover - import sys - import marshal - try: - stdin = sys.stdin.buffer - stdout = sys.stdout.buffer - except AttributeError: - stdin = sys.stdin - stdout = sys.stdout - - (item_depth,) = sys.argv[1:] - item_depth = int(item_depth) - - def handle_item(path, item): - marshal.dump((path, item), stdout) - return True - - try: - root = parse(stdin, - item_depth=item_depth, - item_callback=handle_item, - dict_constructor=dict) - if item_depth == 0: - handle_item([], root) - except KeyboardInterrupt: - pass -- cgit v1.3