diff options
| author | YDBot <[email protected]> | 2025-09-28 05:09:59 +0000 |
|---|---|---|
| committer | YDBot <[email protected]> | 2025-09-28 05:09:59 +0000 |
| commit | 4c4fb159abea49c1893f82da1e45500a7f73b201 (patch) | |
| tree | 74af76bddcb35d8bffb968ff121be647648c6ddf /contrib/python | |
| parent | db154818a7ba839a7dc7b99f28fa5baec43e00ec (diff) | |
| parent | 8dc51f7809129db227c86ccd6a12a56a776d0d8e (diff) | |
Merge pull request #25920 from ydb-platform/merge-rightlib-250928-0051
Diffstat (limited to 'contrib/python')
| -rw-r--r-- | contrib/python/xmltodict/py3/.dist-info/METADATA | 103 | ||||
| -rw-r--r-- | contrib/python/xmltodict/py3/README.md | 101 | ||||
| -rw-r--r-- | contrib/python/xmltodict/py3/tests/test_dicttoxml.py | 286 | ||||
| -rw-r--r-- | contrib/python/xmltodict/py3/tests/test_xmltodict.py | 165 | ||||
| -rw-r--r-- | contrib/python/xmltodict/py3/xmltodict.py | 162 | ||||
| -rw-r--r-- | contrib/python/xmltodict/py3/ya.make | 2 |
6 files changed, 716 insertions, 103 deletions
diff --git a/contrib/python/xmltodict/py3/.dist-info/METADATA b/contrib/python/xmltodict/py3/.dist-info/METADATA index 8f05caf86de..e820bae190d 100644 --- a/contrib/python/xmltodict/py3/.dist-info/METADATA +++ b/contrib/python/xmltodict/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.4 Name: xmltodict -Version: 0.15.1 +Version: 1.0.0 Summary: Makes working with XML feel like you are working with JSON Home-page: https://github.com/martinblech/xmltodict Author: Martin Blech @@ -230,6 +230,95 @@ Lists that are specified under a key in a dictionary use the key as a tag for ea </line> ``` +## API Reference + +### xmltodict.parse() + +Parse XML input into a Python dictionary. + +- `xml_input`: XML input as a string, file-like object, or generator of strings. +- `encoding=None`: Character encoding for the input XML. +- `expat=expat`: XML parser module to use. +- `process_namespaces=False`: Expand XML namespaces if True. +- `namespace_separator=':'`: Separator between namespace URI and local name. +- `disable_entities=True`: Disable entity parsing for security. +- `process_comments=False`: Include XML comments if True. Comments can be preserved when enabled, but by default they are ignored. Multiple top-level comments may not be preserved in exact order. +- `xml_attribs=True`: Include attributes in output dict (with `attr_prefix`). +- `attr_prefix='@'`: Prefix for XML attributes in the dict. +- `cdata_key='#text'`: Key for text content in the dict. +- `force_cdata=False`: Force text content to be wrapped as CDATA for specific elements. Can be a boolean (True/False), a tuple of element names to force CDATA for, or a callable function that receives (path, key, value) and returns True/False. +- `cdata_separator=''`: Separator string to join multiple text nodes. This joins adjacent text nodes. For example, set to a space to avoid concatenation. +- `postprocessor=None`: Function to modify parsed items. +- `dict_constructor=dict`: Constructor for dictionaries (e.g., dict). +- `strip_whitespace=True`: Remove leading/trailing whitespace in text nodes. Default is True; this trims whitespace in text nodes. Set to False to preserve whitespace exactly. +- `namespaces=None`: Mapping of namespaces to prefixes, or None to keep full URIs. +- `force_list=None`: Force list values for specific elements. Can be a boolean (True/False), a tuple of element names to force lists for, or a callable function that receives (path, key, value) and returns True/False. Useful for elements that may appear once or multiple times to ensure consistent list output. +- `item_depth=0`: Depth at which to call `item_callback`. +- `item_callback=lambda *args: True`: Function called on items at `item_depth`. +- `comment_key='#comment'`: Key used for XML comments when `process_comments=True`. Only used when `process_comments=True`. Comments can be preserved but multiple top-level comments may not retain order. + +### xmltodict.unparse() + +Convert a Python dictionary back into XML. + +- `input_dict`: Dictionary to convert to XML. +- `output=None`: File-like object to write XML to; returns string if None. +- `encoding='utf-8'`: Encoding of the output XML. +- `full_document=True`: Include XML declaration if True. +- `short_empty_elements=False`: Use short tags for empty elements (`<tag/>`). +- `attr_prefix='@'`: Prefix for dictionary keys representing attributes. +- `cdata_key='#text'`: Key for text content in the dictionary. +- `pretty=False`: Pretty-print the XML output. +- `indent='\t'`: Indentation string for pretty printing. +- `newl='\n'`: Newline character for pretty printing. +- `expand_iter=None`: Tag name to use for items in nested lists (breaks roundtripping). + +Note: xmltodict aims to cover the common 90% of cases. It does not preserve every XML nuance (attribute order, mixed content ordering, multiple top-level comments). For exact fidelity, use a full XML library such as lxml. + +## Examples + +### Selective force_cdata + +The `force_cdata` parameter can be used to selectively force CDATA wrapping for specific elements: + +```python +>>> xml = '<a><b>data1</b><c>data2</c><d>data3</d></a>' +>>> # Force CDATA only for 'b' and 'd' elements +>>> xmltodict.parse(xml, force_cdata=('b', 'd')) +{'a': {'b': {'#text': 'data1'}, 'c': 'data2', 'd': {'#text': 'data3'}}} + +>>> # Force CDATA for all elements (original behavior) +>>> xmltodict.parse(xml, force_cdata=True) +{'a': {'b': {'#text': 'data1'}, 'c': {'#text': 'data2'}, 'd': {'#text': 'data3'}}} + +>>> # Use a callable for complex logic +>>> def should_force_cdata(path, key, value): +... return key in ['b', 'd'] and len(value) > 4 +>>> xmltodict.parse(xml, force_cdata=should_force_cdata) +{'a': {'b': {'#text': 'data1'}, 'c': 'data2', 'd': {'#text': 'data3'}}} +``` + +### Selective force_list + +The `force_list` parameter can be used to selectively force list values for specific elements: + +```python +>>> xml = '<a><b>data1</b><b>data2</b><c>data3</c></a>' +>>> # Force lists only for 'b' elements +>>> xmltodict.parse(xml, force_list=('b',)) +{'a': {'b': ['data1', 'data2'], 'c': 'data3'}} + +>>> # Force lists for all elements (original behavior) +>>> xmltodict.parse(xml, force_list=True) +{'a': [{'b': ['data1', 'data2'], 'c': ['data3']}]} + +>>> # Use a callable for complex logic +>>> def should_force_list(path, key, value): +... return key in ['b'] and isinstance(value, str) +>>> xmltodict.parse(xml, force_list=should_force_list) +{'a': {'b': ['data1', 'data2'], 'c': 'data3'}} +``` + ## Ok, how do I get it? ### Using pypi @@ -295,6 +384,18 @@ $ zypper in python2-xmltodict $ zypper in python3-xmltodict ``` +## Type Annotations + +For type checking support, install the external types package: + +```sh +# Using pypi +$ pip install types-xmltodict + +# Using conda +$ conda install -c conda-forge types-xmltodict +``` + ## Security Notes A CVE (CVE-2025-9375) was filed against `xmltodict` but is [disputed](https://github.com/martinblech/xmltodict/issues/377#issuecomment-3255691923). The root issue lies in Python’s `xml.sax.saxutils.XMLGenerator` API, which does not validate XML element names and provides no built-in way to do so. Since `xmltodict` is a thin wrapper that passes keys directly to `XMLGenerator`, the same issue exists in the standard library itself. diff --git a/contrib/python/xmltodict/py3/README.md b/contrib/python/xmltodict/py3/README.md index 4c24cf100f0..dec9d9bdabf 100644 --- a/contrib/python/xmltodict/py3/README.md +++ b/contrib/python/xmltodict/py3/README.md @@ -194,6 +194,95 @@ Lists that are specified under a key in a dictionary use the key as a tag for ea </line> ``` +## API Reference + +### xmltodict.parse() + +Parse XML input into a Python dictionary. + +- `xml_input`: XML input as a string, file-like object, or generator of strings. +- `encoding=None`: Character encoding for the input XML. +- `expat=expat`: XML parser module to use. +- `process_namespaces=False`: Expand XML namespaces if True. +- `namespace_separator=':'`: Separator between namespace URI and local name. +- `disable_entities=True`: Disable entity parsing for security. +- `process_comments=False`: Include XML comments if True. Comments can be preserved when enabled, but by default they are ignored. Multiple top-level comments may not be preserved in exact order. +- `xml_attribs=True`: Include attributes in output dict (with `attr_prefix`). +- `attr_prefix='@'`: Prefix for XML attributes in the dict. +- `cdata_key='#text'`: Key for text content in the dict. +- `force_cdata=False`: Force text content to be wrapped as CDATA for specific elements. Can be a boolean (True/False), a tuple of element names to force CDATA for, or a callable function that receives (path, key, value) and returns True/False. +- `cdata_separator=''`: Separator string to join multiple text nodes. This joins adjacent text nodes. For example, set to a space to avoid concatenation. +- `postprocessor=None`: Function to modify parsed items. +- `dict_constructor=dict`: Constructor for dictionaries (e.g., dict). +- `strip_whitespace=True`: Remove leading/trailing whitespace in text nodes. Default is True; this trims whitespace in text nodes. Set to False to preserve whitespace exactly. +- `namespaces=None`: Mapping of namespaces to prefixes, or None to keep full URIs. +- `force_list=None`: Force list values for specific elements. Can be a boolean (True/False), a tuple of element names to force lists for, or a callable function that receives (path, key, value) and returns True/False. Useful for elements that may appear once or multiple times to ensure consistent list output. +- `item_depth=0`: Depth at which to call `item_callback`. +- `item_callback=lambda *args: True`: Function called on items at `item_depth`. +- `comment_key='#comment'`: Key used for XML comments when `process_comments=True`. Only used when `process_comments=True`. Comments can be preserved but multiple top-level comments may not retain order. + +### xmltodict.unparse() + +Convert a Python dictionary back into XML. + +- `input_dict`: Dictionary to convert to XML. +- `output=None`: File-like object to write XML to; returns string if None. +- `encoding='utf-8'`: Encoding of the output XML. +- `full_document=True`: Include XML declaration if True. +- `short_empty_elements=False`: Use short tags for empty elements (`<tag/>`). +- `attr_prefix='@'`: Prefix for dictionary keys representing attributes. +- `cdata_key='#text'`: Key for text content in the dictionary. +- `pretty=False`: Pretty-print the XML output. +- `indent='\t'`: Indentation string for pretty printing. +- `newl='\n'`: Newline character for pretty printing. +- `expand_iter=None`: Tag name to use for items in nested lists (breaks roundtripping). + +Note: xmltodict aims to cover the common 90% of cases. It does not preserve every XML nuance (attribute order, mixed content ordering, multiple top-level comments). For exact fidelity, use a full XML library such as lxml. + +## Examples + +### Selective force_cdata + +The `force_cdata` parameter can be used to selectively force CDATA wrapping for specific elements: + +```python +>>> xml = '<a><b>data1</b><c>data2</c><d>data3</d></a>' +>>> # Force CDATA only for 'b' and 'd' elements +>>> xmltodict.parse(xml, force_cdata=('b', 'd')) +{'a': {'b': {'#text': 'data1'}, 'c': 'data2', 'd': {'#text': 'data3'}}} + +>>> # Force CDATA for all elements (original behavior) +>>> xmltodict.parse(xml, force_cdata=True) +{'a': {'b': {'#text': 'data1'}, 'c': {'#text': 'data2'}, 'd': {'#text': 'data3'}}} + +>>> # Use a callable for complex logic +>>> def should_force_cdata(path, key, value): +... return key in ['b', 'd'] and len(value) > 4 +>>> xmltodict.parse(xml, force_cdata=should_force_cdata) +{'a': {'b': {'#text': 'data1'}, 'c': 'data2', 'd': {'#text': 'data3'}}} +``` + +### Selective force_list + +The `force_list` parameter can be used to selectively force list values for specific elements: + +```python +>>> xml = '<a><b>data1</b><b>data2</b><c>data3</c></a>' +>>> # Force lists only for 'b' elements +>>> xmltodict.parse(xml, force_list=('b',)) +{'a': {'b': ['data1', 'data2'], 'c': 'data3'}} + +>>> # Force lists for all elements (original behavior) +>>> xmltodict.parse(xml, force_list=True) +{'a': [{'b': ['data1', 'data2'], 'c': ['data3']}]} + +>>> # Use a callable for complex logic +>>> def should_force_list(path, key, value): +... return key in ['b'] and isinstance(value, str) +>>> xmltodict.parse(xml, force_list=should_force_list) +{'a': {'b': ['data1', 'data2'], 'c': 'data3'}} +``` + ## Ok, how do I get it? ### Using pypi @@ -259,6 +348,18 @@ $ zypper in python2-xmltodict $ zypper in python3-xmltodict ``` +## Type Annotations + +For type checking support, install the external types package: + +```sh +# Using pypi +$ pip install types-xmltodict + +# Using conda +$ conda install -c conda-forge types-xmltodict +``` + ## Security Notes A CVE (CVE-2025-9375) was filed against `xmltodict` but is [disputed](https://github.com/martinblech/xmltodict/issues/377#issuecomment-3255691923). The root issue lies in Python’s `xml.sax.saxutils.XMLGenerator` API, which does not validate XML element names and provides no built-in way to do so. Since `xmltodict` is a thin wrapper that passes keys directly to `XMLGenerator`, the same issue exists in the standard library itself. diff --git a/contrib/python/xmltodict/py3/tests/test_dicttoxml.py b/contrib/python/xmltodict/py3/tests/test_dicttoxml.py index 1fa5ba78316..87c3d256714 100644 --- a/contrib/python/xmltodict/py3/tests/test_dicttoxml.py +++ b/contrib/python/xmltodict/py3/tests/test_dicttoxml.py @@ -1,13 +1,9 @@ -import sys from xmltodict import parse, unparse -from collections import OrderedDict import unittest import re from textwrap import dedent -IS_JYTHON = sys.platform.startswith('java') - _HEADER_RE = re.compile(r'^[^\n]*\n') @@ -74,7 +70,7 @@ class DictToXMLTestCase(unittest.TestCase): self.assertEqual(unparse({}, full_document=False), '') def test_multiple_roots_nofulldoc(self): - obj = OrderedDict((('a', 1), ('b', 2))) + obj = {"a": 1, "b": 2} xml = unparse(obj, full_document=False) self.assertEqual(xml, '<a>1</a><b>2</b>') obj = {'a': [1, 2]} @@ -95,7 +91,7 @@ class DictToXMLTestCase(unittest.TestCase): '<a><d></d>abcefg</a>') def test_preprocessor(self): - obj = {'a': OrderedDict((('b:int', [1, 2]), ('b', 'c')))} + obj = {"a": {"b:int": [1, 2], "b": "c"}} def p(key, value): try: @@ -118,17 +114,17 @@ class DictToXMLTestCase(unittest.TestCase): self.assertEqual(_strip(unparse(obj, preprocessor=p)), '<a><c>2</c></a>') - if not IS_JYTHON: - # Jython's SAX does not preserve attribute order - def test_attr_order_roundtrip(self): - xml = '<root a="1" b="2" c="3"></root>' - self.assertEqual(xml, _strip(unparse(parse(xml)))) + def test_attr_order_roundtrip(self): + xml = '<root a="1" b="2" c="3"></root>' + self.assertEqual(xml, _strip(unparse(parse(xml)))) def test_pretty_print(self): - obj = {'a': OrderedDict(( - ('b', [{'c': [1, 2]}, 3]), - ('x', 'y'), - ))} + obj = { + "a": { + "b": [{"c": [1, 2]}, 3], + "x": "y", + } + } newl = '\n' indent = '....' xml = dedent('''\ @@ -144,11 +140,33 @@ class DictToXMLTestCase(unittest.TestCase): self.assertEqual(xml, unparse(obj, pretty=True, newl=newl, indent=indent)) + def test_unparse_with_element_comment(self): + obj = {"a": {"#comment": "note", "b": "1"}} + xml = _strip(unparse(obj, full_document=True)) + self.assertEqual(xml, "<a><!--note--><b>1</b></a>") + + def test_unparse_with_multiple_element_comments(self): + obj = {"a": {"#comment": ["n1", "n2"], "b": "1"}} + xml = _strip(unparse(obj, full_document=True)) + self.assertEqual(xml, "<a><!--n1--><!--n2--><b>1</b></a>") + + def test_unparse_with_top_level_comment(self): + obj = {"#comment": "top", "a": "1"} + xml = _strip(unparse(obj, full_document=True)) + self.assertEqual(xml, "<!--top--><a>1</a>") + + def test_unparse_with_multiple_top_level_comments(self): + obj = {"#comment": ["t1", "t2"], "a": "1"} + xml = _strip(unparse(obj, full_document=True)) + self.assertEqual(xml, "<!--t1--><!--t2--><a>1</a>") + def test_pretty_print_with_int_indent(self): - obj = {'a': OrderedDict(( - ('b', [{'c': [1, 2]}, 3]), - ('x', 'y'), - ))} + obj = { + "a": { + "b": [{"c": [1, 2]}, 3], + "x": "y", + } + } newl = '\n' indent = 2 xml = dedent('''\ @@ -164,11 +182,32 @@ class DictToXMLTestCase(unittest.TestCase): self.assertEqual(xml, unparse(obj, pretty=True, newl=newl, indent=indent)) + def test_comment_roundtrip_limited(self): + # Input with top-level comments and an element-level comment + xml = """ + <!--top1--><a><b>1</b><!--e1--></a><!--top2--> + """ + # Parse with comment processing enabled + parsed1 = parse(xml, process_comments=True) + # Unparse and parse again (roundtrip) + xml2 = unparse(parsed1) + parsed2 = parse(xml2, process_comments=True) + + # Content preserved + self.assertIn('a', parsed2) + self.assertEqual(parsed2['a']['b'], '1') + + # Element-level comment preserved under '#comment' + self.assertEqual(parsed2['a']['#comment'], 'e1') + + # Top-level comments preserved as a list (order not guaranteed) + top = parsed2.get('#comment') + self.assertIsNotNone(top) + top_list = top if isinstance(top, list) else [top] + self.assertEqual(set(top_list), {'top1', 'top2'}) + def test_encoding(self): - try: - value = unichr(39321) - except NameError: - value = chr(39321) + value = chr(39321) obj = {'a': value} utf8doc = unparse(obj, encoding='utf-8') latin1doc = unparse(obj, encoding='iso-8859-1') @@ -195,21 +234,21 @@ class DictToXMLTestCase(unittest.TestCase): self.assertEqual('<a/>', _strip(unparse(obj, short_empty_elements=True))) def test_namespace_support(self): - obj = OrderedDict(( - ('http://defaultns.com/:root', OrderedDict(( - ('@xmlns', OrderedDict(( - ('', 'http://defaultns.com/'), - ('a', 'http://a.com/'), - ('b', 'http://b.com/'), - ))), - ('http://defaultns.com/:x', OrderedDict(( - ('@http://a.com/:attr', 'val'), - ('#text', '1'), - ))), - ('http://a.com/:y', '2'), - ('http://b.com/:z', '3'), - ))), - )) + obj = { + "http://defaultns.com/:root": { + "@xmlns": { + "": "http://defaultns.com/", + "a": "http://a.com/", + "b": "http://b.com/", + }, + "http://defaultns.com/:x": { + "@http://a.com/:attr": "val", + "#text": "1", + }, + "http://a.com/:y": "2", + "http://b.com/:z": "3", + }, + } ns = { 'http://defaultns.com/': '', 'http://a.com/': 'a', @@ -337,3 +376,174 @@ xmlns:b="http://b.com/"><x a:attr="val">1</x><a:y>2</a:y><b:z>3</b:z></root>''' for prefix in ['a"b', "a'b", "a=b"]: with self.assertRaises(ValueError): unparse({"a": {"@xmlns": {prefix: "http://e/"}}}, full_document=False) + + def test_pretty_print_and_short_empty_elements_consistency(self): + """Test that pretty and compact modes produce equivalent results when stripped. + + This test covers issue #352: Edge case with pretty_print and short_empty_elements. + When short_empty_elements=True, empty elements should be written as <tag/> + regardless of whether pretty printing is enabled. + """ + # Test case from issue #352: empty list child + input_dict = {"Foos": {"Foo": []}} + + compact = unparse( + input_dict, pretty=False, short_empty_elements=True, full_document=False + ) + pretty = unparse( + input_dict, pretty=True, short_empty_elements=True, full_document=False + ) + pretty_compacted = pretty.replace("\n", "").replace("\t", "") + + # They should be equal when pretty formatting is stripped + self.assertEqual(pretty_compacted, compact) + self.assertEqual(compact, "<Foos/>") + self.assertEqual(pretty_compacted, "<Foos/>") + + def test_empty_list_filtering(self): + """Test that empty lists are filtered out and don't create empty child elements.""" + # Test various cases with empty lists + test_cases = [ + # Case 1: Single empty list child + ({"Foos": {"Foo": []}}, "<Foos/>"), + # Case 2: Multiple empty list children + ({"Foos": {"Foo": [], "Bar": []}}, "<Foos/>"), + # Case 3: Mixed empty and non-empty children + ({"Foos": {"Foo": [], "Bar": "value"}}, "<Foos><Bar>value</Bar></Foos>"), + # Case 4: Nested empty lists + ({"Foos": {"Foo": {"Bar": []}}}, "<Foos><Foo/></Foos>"), + # Case 5: Empty list with attributes + ({"Foos": {"@attr": "value", "Foo": []}}, '<Foos attr="value"/>'), + ] + + for input_dict, expected_compact in test_cases: + with self.subTest(input_dict=input_dict): + # Test compact mode + compact = unparse( + input_dict, + pretty=False, + short_empty_elements=True, + full_document=False, + ) + self.assertEqual(compact, expected_compact) + + # Test pretty mode + pretty = unparse( + input_dict, + pretty=True, + short_empty_elements=True, + full_document=False, + ) + pretty_compacted = pretty.replace("\n", "").replace("\t", "") + self.assertEqual(pretty_compacted, expected_compact) + + def test_empty_list_filtering_with_short_empty_elements_false(self): + """Test that empty lists are still filtered when short_empty_elements=False.""" + input_dict = {"Foos": {"Foo": []}} + + # With short_empty_elements=False, empty elements should be <tag></tag> + compact = unparse( + input_dict, pretty=False, short_empty_elements=False, full_document=False + ) + pretty = unparse( + input_dict, pretty=True, short_empty_elements=False, full_document=False + ) + pretty_compacted = pretty.replace("\n", "").replace("\t", "") + + # They should be equal when pretty formatting is stripped + self.assertEqual(pretty_compacted, compact) + self.assertEqual(compact, "<Foos></Foos>") + self.assertEqual(pretty_compacted, "<Foos></Foos>") + + def test_non_empty_lists_are_not_filtered(self): + """Test that non-empty lists are not filtered out.""" + # Test with non-empty lists + input_dict = {"Foos": {"Foo": ["item1", "item2"]}} + + compact = unparse( + input_dict, pretty=False, short_empty_elements=True, full_document=False + ) + pretty = unparse( + input_dict, pretty=True, short_empty_elements=True, full_document=False + ) + pretty_compacted = pretty.replace("\n", "").replace("\t", "") + + # The lists should be processed normally + self.assertEqual(pretty_compacted, compact) + self.assertEqual(compact, "<Foos><Foo>item1</Foo><Foo>item2</Foo></Foos>") + self.assertEqual( + pretty_compacted, "<Foos><Foo>item1</Foo><Foo>item2</Foo></Foos>" + ) + + def test_empty_dict_vs_empty_list_behavior(self): + """Test the difference between empty dicts and empty lists.""" + # Empty dict should create a child element + input_dict_dict = {"Foos": {"Foo": {}}} + compact_dict = unparse( + input_dict_dict, + pretty=False, + short_empty_elements=True, + full_document=False, + ) + self.assertEqual(compact_dict, "<Foos><Foo/></Foos>") + + # Empty list should be filtered out + input_dict_list = {"Foos": {"Foo": []}} + compact_list = unparse( + input_dict_list, + pretty=False, + short_empty_elements=True, + full_document=False, + ) + self.assertEqual(compact_list, "<Foos/>") + + # They should be different + self.assertNotEqual(compact_dict, compact_list) + + def test_non_string_text_with_attributes(self): + """Test that non-string #text values work when tag has attributes. + + This test covers GitHub issue #366: Tag value (#text) must be a string + when tag has additional parameters - unparse. + + Also tests that plain values and explicit #text values are treated + consistently (both go through the same conversion logic). + """ + # Test cases for explicit #text values with attributes + self.assertEqual(unparse({"a": {"@param": "test", "#text": 1}}, full_document=False), + '<a param="test">1</a>') + + self.assertEqual(unparse({"a": {"@param": 42, "#text": 3.14}}, full_document=False), + '<a param="42">3.14</a>') + + self.assertEqual(unparse({"a": {"@param": "flag", "#text": True}}, full_document=False), + '<a param="flag">true</a>') + + self.assertEqual(unparse({"a": {"@param": "test", "#text": None}}, full_document=False), + '<a param="test">None</a>') + + self.assertEqual(unparse({"a": {"@param": "test", "#text": "string"}}, full_document=False), + '<a param="test">string</a>') + + self.assertEqual(unparse({"a": {"@attr1": "value1", "@attr2": 2, "#text": 100}}, full_document=False), + '<a attr1="value1" attr2="2">100</a>') + + # Test cases for plain values (should be treated the same as #text) + self.assertEqual(unparse({"a": 1}, full_document=False), '<a>1</a>') + self.assertEqual(unparse({"a": 3.14}, full_document=False), '<a>3.14</a>') + self.assertEqual(unparse({"a": True}, full_document=False), '<a>true</a>') + self.assertEqual(unparse({"a": "hello"}, full_document=False), '<a>hello</a>') + self.assertEqual(unparse({"a": None}, full_document=False), '<a></a>') + + # Consistency tests: plain values should match explicit #text values + self.assertEqual(unparse({"a": 42}, full_document=False), + unparse({"a": {"#text": 42}}, full_document=False)) + + self.assertEqual(unparse({"a": 3.14}, full_document=False), + unparse({"a": {"#text": 3.14}}, full_document=False)) + + self.assertEqual(unparse({"a": True}, full_document=False), + unparse({"a": {"#text": True}}, full_document=False)) + + self.assertEqual(unparse({"a": "hello"}, full_document=False), + unparse({"a": {"#text": "hello"}}, full_document=False)) diff --git a/contrib/python/xmltodict/py3/tests/test_xmltodict.py b/contrib/python/xmltodict/py3/tests/test_xmltodict.py index 7dd22b53f3b..5c2bbe7d243 100644 --- a/contrib/python/xmltodict/py3/tests/test_xmltodict.py +++ b/contrib/python/xmltodict/py3/tests/test_xmltodict.py @@ -39,6 +39,68 @@ class XMLToDictTestCase(unittest.TestCase): self.assertEqual(parse('<a>data</a>', force_cdata=True), {'a': {'#text': 'data'}}) + def test_selective_force_cdata_tuple(self): + xml = "<a><b>data1</b><c>data2</c><d>data3</d></a>" + # Test with tuple of specific element names + result = parse(xml, force_cdata=("b", "d")) + expected = { + "a": {"b": {"#text": "data1"}, "c": "data2", "d": {"#text": "data3"}} + } + self.assertEqual(result, expected) + + def test_selective_force_cdata_single_element(self): + xml = "<a><b>data1</b><c>data2</c></a>" + # Test with single element name + result = parse(xml, force_cdata=("b",)) + expected = {"a": {"b": {"#text": "data1"}, "c": "data2"}} + self.assertEqual(result, expected) + + def test_selective_force_cdata_empty_tuple(self): + xml = "<a><b>data1</b><c>data2</c></a>" + # Test with empty tuple (should behave like force_cdata=False) + result = parse(xml, force_cdata=()) + expected = {"a": {"b": "data1", "c": "data2"}} + self.assertEqual(result, expected) + + def test_selective_force_cdata_callable(self): + xml = "<a><b>data1</b><c>data2</c><d>data3</d></a>" + + # Test with callable function + def should_force_cdata(path, key, value): + return key in ["b", "d"] + + result = parse(xml, force_cdata=should_force_cdata) + expected = { + "a": {"b": {"#text": "data1"}, "c": "data2", "d": {"#text": "data3"}} + } + self.assertEqual(result, expected) + + def test_selective_force_cdata_nested_elements(self): + xml = "<a><b><c>data1</c></b><d>data2</d></a>" + # Test with nested elements - only 'c' should be forced + result = parse(xml, force_cdata=("c",)) + expected = {"a": {"b": {"c": {"#text": "data1"}}, "d": "data2"}} + self.assertEqual(result, expected) + + def test_selective_force_cdata_with_attributes(self): + xml = '<a><b attr="value">data1</b><c>data2</c></a>' + # Test with attributes - force_cdata should still work + result = parse(xml, force_cdata=("b",)) + expected = {"a": {"b": {"@attr": "value", "#text": "data1"}, "c": "data2"}} + self.assertEqual(result, expected) + + def test_selective_force_cdata_backwards_compatibility(self): + xml = "<a><b>data1</b><c>data2</c></a>" + # Test that boolean True still works (backwards compatibility) + result_true = parse(xml, force_cdata=True) + expected_true = {"a": {"b": {"#text": "data1"}, "c": {"#text": "data2"}}} + self.assertEqual(result_true, expected_true) + + # Test that boolean False still works (backwards compatibility) + result_false = parse(xml, force_cdata=False) + expected_false = {"a": {"b": "data1", "c": "data2"}} + self.assertEqual(result_false, expected_false) + def test_custom_cdata(self): self.assertEqual(parse('<a>data</a>', force_cdata=True, @@ -115,7 +177,8 @@ class XMLToDictTestCase(unittest.TestCase): self.assertEqual(cb.count, 3) def test_streaming_interrupt(self): - cb = lambda path, item: False + def cb(path, item): + return False self.assertRaises(ParsingInterrupted, parse, '<a>x</a>', item_depth=1, item_callback=cb) @@ -131,6 +194,14 @@ class XMLToDictTestCase(unittest.TestCase): item_depth=2, item_callback=cb) self.assertEqual(cb.count, 3) + def test_streaming_returns_none(self): + # When streaming (item_depth > 0), parse should return None + def cb(path, item): + return True + + result = parse("<a><b>1</b><b>2</b></a>", item_depth=2, item_callback=cb) + self.assertIsNone(result) + def test_postprocessor(self): def postprocessor(path, key, value): try: @@ -163,18 +234,12 @@ class XMLToDictTestCase(unittest.TestCase): postprocessor=postprocessor)) def test_unicode(self): - try: - value = unichr(39321) - except NameError: - value = chr(39321) + value = chr(39321) self.assertEqual({'a': value}, parse(f'<a>{value}</a>')) def test_encoded_string(self): - try: - value = unichr(39321) - except NameError: - value = chr(39321) + value = chr(39321) xml = f'<a>{value}</a>' self.assertEqual(parse(xml), parse(xml.encode('utf-8'))) @@ -421,7 +486,7 @@ class XMLToDictTestCase(unittest.TestCase): return parser expat.ParserCreate = raising_external_ref_handler # Using this try/catch because a TypeError is thrown before - # the ExpatError, and Python 2.6 is confused by that. + # the ExpatError. try: parse(xml, disable_entities=False, expat=expat) except expat.ExpatError: @@ -458,6 +523,61 @@ class XMLToDictTestCase(unittest.TestCase): } self.assertEqual(parse(xml, process_comments=True), expectedResult) + def test_streaming_with_comments_and_attrs(self): + xml = """ + <a> + <b attr1="value"> + <!-- note --> + <c>cdata</c> + </b> + </a> + """ + + def handler(path, item): + expected = { + "@attr1": "value", + "#comment": "note", + "c": "cdata", + } + self.assertEqual(expected, item) + return True + + parse(xml, item_depth=2, item_callback=handler, process_comments=True) + + def test_streaming_memory_usage(self): + # Guard against re-introducing accumulation of streamed items into parent + try: + import tracemalloc + except ImportError: + self.skipTest("tracemalloc not available") + + NUM_ITEMS = 20000 + + def xml_gen(): + yield "<a>" + # generate many children with attribute and text + for i in range(NUM_ITEMS): + yield f'<b attr="v">{i % 10}</b>' + yield "</a>" + + count = 0 + + def cb(path, item): + nonlocal count + count += 1 + return True + + tracemalloc.start() + parse(xml_gen(), item_depth=2, item_callback=cb) + current, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + + self.assertEqual(count, NUM_ITEMS) + # Peak memory should remain reasonably bounded; choose a conservative threshold + # This value should stay well below pathological accumulation levels + MAX_BYTES = 32 * 1024 # 32 KiB + self.assertLess(peak, MAX_BYTES, f"peak memory too high: {peak} bytes") + def test_streaming_attrs(self): xml = """ <a> @@ -475,3 +595,28 @@ class XMLToDictTestCase(unittest.TestCase): return True parse(xml, item_depth=2, item_callback=handler) + + def test_namespace_on_root_without_other_attrs(self): + xml = """ + <MyXML xmlns="http://www.xml.org/schemas/Test"> + <Tag1>Text1</Tag1> + <Tag2 attr2="en">Text2</Tag2> + <Tag3>Text3</Tag3> + <Tag4 attr4="en">Text4</Tag4> + </MyXML> + """ + namespaces = { + "http://www.xml.org/schemas/Test": None, + } + expected = { + "MyXML": { + "@xmlns": {"": "http://www.xml.org/schemas/Test"}, + "Tag1": "Text1", + "Tag2": {"@attr2": "en", "#text": "Text2"}, + "Tag3": "Text3", + "Tag4": {"@attr4": "en", "#text": "Text4"}, + } + } + self.assertEqual( + parse(xml, process_namespaces=True, namespaces=namespaces), expected + ) diff --git a/contrib/python/xmltodict/py3/xmltodict.py b/contrib/python/xmltodict/py3/xmltodict.py index 4b6852ab23c..2735ad71c45 100644 --- a/contrib/python/xmltodict/py3/xmltodict.py +++ b/contrib/python/xmltodict/py3/xmltodict.py @@ -2,19 +2,13 @@ "Makes working with XML feel like you are working with JSON" from xml.parsers import expat -from xml.sax.saxutils import XMLGenerator +from xml.sax.saxutils import XMLGenerator, escape from xml.sax.xmlreader import AttributesImpl from io import StringIO - -_dict = dict -import platform -if tuple(map(int, platform.python_version_tuple()[:2])) < (3, 7): - from collections import OrderedDict as _dict - from inspect import isgenerator __author__ = 'Martin Blech' -__version__ = "0.15.1" +__version__ = "1.0.0" # x-release-please-version __license__ = 'MIT' @@ -23,21 +17,23 @@ class ParsingInterrupted(Exception): class _DictSAXHandler: - def __init__(self, - item_depth=0, - item_callback=lambda *args: True, - xml_attribs=True, - attr_prefix='@', - cdata_key='#text', - force_cdata=False, - cdata_separator='', - postprocessor=None, - dict_constructor=_dict, - strip_whitespace=True, - namespace_separator=':', - namespaces=None, - force_list=None, - comment_key='#comment'): + def __init__( + self, + item_depth=0, + item_callback=lambda *args: True, + xml_attribs=True, + attr_prefix="@", + cdata_key="#text", + force_cdata=False, + cdata_separator="", + postprocessor=None, + dict_constructor=dict, + strip_whitespace=True, + namespace_separator=":", + namespaces=None, + force_list=None, + comment_key="#comment", + ): self.path = [] self.stack = [] self.data = [] @@ -85,7 +81,9 @@ class _DictSAXHandler: def startElement(self, full_name, attrs): name = self._build_name(full_name) attrs = self._attrs_to_dict(attrs) - if attrs and self.namespace_declarations: + if self.namespace_declarations: + if not attrs: + attrs = self.dict_constructor() attrs['xmlns'] = self.namespace_declarations self.namespace_declarations = self.dict_constructor() self.path.append((name, attrs or None)) @@ -109,6 +107,9 @@ class _DictSAXHandler: def endElement(self, full_name): name = self._build_name(full_name) + # If we just closed an item at the streaming depth, emit it and drop it + # without attaching it back to its parent. This avoids accumulating all + # streamed items in memory when using item_depth > 0. if len(self.path) == self.item_depth: item = self.item if item is None: @@ -118,6 +119,15 @@ class _DictSAXHandler: should_continue = self.item_callback(self.path, item) if not should_continue: raise ParsingInterrupted + # Reset state for the parent context without keeping a reference to + # the emitted item. + if self.stack: + self.item, self.data = self.stack.pop() + else: + self.item = None + self.data = [] + self.path.pop() + return if self.stack: data = (None if not self.data else self.cdata_separator.join(self.data)) @@ -125,7 +135,7 @@ class _DictSAXHandler: self.item, self.data = self.stack.pop() if self.strip_whitespace and data: data = data.strip() or None - if data and self.force_cdata and item is None: + if data and self._should_force_cdata(name, data) and item is None: item = self.dict_constructor() if item is not None: if data: @@ -180,6 +190,16 @@ class _DictSAXHandler: except TypeError: return self.force_list(self.path[:-1], key, value) + def _should_force_cdata(self, key, value): + if not self.force_cdata: + return False + if isinstance(self.force_cdata, bool): + return self.force_cdata + try: + return key in self.force_cdata + except TypeError: + return self.force_cdata(self.path[:-1], key, value) + def parse(xml_input, encoding=None, expat=expat, process_namespaces=False, namespace_separator=':', disable_entities=True, process_comments=False, **kwargs): @@ -201,9 +221,9 @@ def parse(xml_input, encoding=None, expat=expat, process_namespaces=False, ... </a> ... \"\"\") >>> doc['a']['@prop'] - u'x' + 'x' >>> doc['a']['b'] - [u'1', u'2'] + ['1', '2'] If `item_depth` is `0`, the function returns a dictionary for the root element (default behavior). Otherwise, it calls `item_callback` every time @@ -226,8 +246,8 @@ def parse(xml_input, encoding=None, expat=expat, process_namespaces=False, ... <b>1</b> ... <b>2</b> ... </a>\"\"\", item_depth=2, item_callback=handle) - path:[(u'a', {u'prop': u'x'}), (u'b', None)] item:1 - path:[(u'a', {u'prop': u'x'}), (u'b', None)] item:2 + path:[('a', {'prop': 'x'}), ('b', None)] item:1 + path:[('a', {'prop': 'x'}), ('b', None)] item:2 The optional argument `postprocessor` is a function that takes `path`, `key` and `value` as positional arguments and returns a new `(key, value)` @@ -360,6 +380,18 @@ def parse(xml_input, encoding=None, expat=expat, process_namespaces=False, return handler.item +def _convert_value_to_string(value): + """Convert a value to its string representation for XML output. + + Handles boolean values consistently by converting them to lowercase. + """ + if isinstance(value, (str, bytes)): + return value + if isinstance(value, bool): + return "true" if value else "false" + return str(value) + + def _has_angle_brackets(value): """Return True if value (a str) contains '<' or '>'. @@ -433,7 +465,25 @@ def _emit(key, value, content_handler, namespace_separator=':', namespaces=None, full_document=True, - expand_iter=None): + expand_iter=None, + comment_key='#comment'): + if isinstance(key, str) and key == comment_key: + comments_list = value if isinstance(value, list) else [value] + if isinstance(indent, int): + indent = " " * indent + for comment_text in comments_list: + if comment_text is None: + continue + comment_text = _convert_value_to_string(comment_text) + if comment_text == "": + continue + if pretty: + content_handler.ignorableWhitespace(depth * indent) + content_handler.comment(comment_text) + if pretty: + content_handler.ignorableWhitespace(newl) + return + key = _process_namespace(key, namespaces, namespace_separator, attr_prefix) if preprocessor is not None: result = preprocessor(key, value) @@ -448,22 +498,20 @@ def _emit(key, value, content_handler, if full_document and depth == 0 and index > 0: raise ValueError('document with multiple roots') if v is None: - v = _dict() - elif isinstance(v, bool): - v = 'true' if v else 'false' + v = {} elif not isinstance(v, (dict, str)): if expand_iter and hasattr(v, '__iter__'): - v = _dict(((expand_iter, v),)) + v = {expand_iter: v} else: - v = str(v) + v = _convert_value_to_string(v) if isinstance(v, str): - v = _dict(((cdata_key, v),)) + v = {cdata_key: v} cdata = None - attrs = _dict() + attrs = {} children = [] for ik, iv in v.items(): if ik == cdata_key: - cdata = iv + cdata = _convert_value_to_string(iv) continue if isinstance(ik, str) and ik.startswith(attr_prefix): ik = _process_namespace(ik, namespaces, namespace_separator, @@ -480,6 +528,8 @@ def _emit(key, value, content_handler, _validate_name(attr_name, "attribute") attrs[attr_name] = iv continue + if isinstance(iv, list) and not iv: + continue # Skip empty lists to avoid creating empty child elements children.append((ik, iv)) if isinstance(indent, int): indent = ' ' * indent @@ -493,7 +543,7 @@ def _emit(key, value, content_handler, attr_prefix, cdata_key, depth+1, preprocessor, pretty, newl, indent, namespaces=namespaces, namespace_separator=namespace_separator, - expand_iter=expand_iter) + expand_iter=expand_iter, comment_key=comment_key) if cdata is not None: content_handler.characters(cdata) if pretty and children: @@ -503,8 +553,13 @@ def _emit(key, value, content_handler, content_handler.ignorableWhitespace(newl) +class _XMLGenerator(XMLGenerator): + def comment(self, text): + self._write(f"<!--{escape(text)}-->") + + def unparse(input_dict, output=None, encoding='utf-8', full_document=True, - short_empty_elements=False, + short_empty_elements=False, comment_key='#comment', **kwargs): """Emit an XML document for the given `input_dict` (reverse of `parse`). @@ -520,21 +575,25 @@ def unparse(input_dict, output=None, encoding='utf-8', full_document=True, can be customized with the `newl` and `indent` parameters. """ - if full_document and len(input_dict) != 1: - raise ValueError('Document must have exactly one root.') must_return = False if output is None: output = StringIO() must_return = True if short_empty_elements: - content_handler = XMLGenerator(output, encoding, True) + content_handler = _XMLGenerator(output, encoding, True) else: - content_handler = XMLGenerator(output, encoding) + content_handler = _XMLGenerator(output, encoding) if full_document: content_handler.startDocument() + seen_root = False for key, value in input_dict.items(): - _emit(key, value, content_handler, full_document=full_document, - **kwargs) + if key != comment_key and full_document and seen_root: + raise ValueError("Document must have exactly one root.") + _emit(key, value, content_handler, full_document=full_document, comment_key=comment_key, **kwargs) + if key != comment_key: + seen_root = True + if full_document and not seen_root: + raise ValueError("Document must have exactly one root.") if full_document: content_handler.endDocument() if must_return: @@ -547,14 +606,11 @@ def unparse(input_dict, output=None, encoding='utf-8', full_document=True, if __name__ == '__main__': # pragma: no cover - import sys import marshal - try: - stdin = sys.stdin.buffer - stdout = sys.stdout.buffer - except AttributeError: - stdin = sys.stdin - stdout = sys.stdout + import sys + + stdin = sys.stdin.buffer + stdout = sys.stdout.buffer (item_depth,) = sys.argv[1:] item_depth = int(item_depth) diff --git a/contrib/python/xmltodict/py3/ya.make b/contrib/python/xmltodict/py3/ya.make index 19f2aa40d76..b9534f46bc6 100644 --- a/contrib/python/xmltodict/py3/ya.make +++ b/contrib/python/xmltodict/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(0.15.1) +VERSION(1.0.0) LICENSE(MIT) |
