diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 4f2497c8246be30..bbe621fca14cf70 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -188,9 +188,17 @@ remove the processed children from it:: These examples are not universal, they only give an idea for two common cases. -If you do not need a tree at all, -parse with :class:`XMLParser` and a custom target instead; -it is not built then, and nothing has to be removed. +If you do not need the tree, +parse with the :class:`XMLPullTarget` target instead: +it reports the elements without building a tree, +and :meth:`XMLPullParser.expand` builds the subtree of an element +when it is needed:: + + it = ET.iterparse(source, events=('start', 'end'), target=ET.XMLPullTarget()) + for event, elem in it: + if event == 'start' and elem.tag == 'record': + it.expand(elem) + process(elem) Where *immediate* feedback through events is wanted, calling method :meth:`XMLPullParser.flush` can help reduce delay; @@ -1332,6 +1340,60 @@ QName Objects +.. _elementtree-xmlpulltarget-objects: + +XMLPullTarget Objects +^^^^^^^^^^^^^^^^^^^^^ + + +.. class:: XMLPullTarget(element_factory=None) + + A target of :class:`XMLParser` which reports elements + without building a tree. + It can be used with :class:`XMLPullParser` and :func:`iterparse`, + which report the objects returned by its methods. + The reported object is an :class:`Element` with the tag, the attributes + and, for the ``"end"`` event, the text, but without children: + they are reported as their own events. + Nothing is kept, so a document of any size can be parsed + with a constant amount of memory. + + *element_factory*, when given, is called to create new elements, + like the argument of :class:`TreeBuilder` with the same name. + + Use :meth:`XMLPullParser.expand` to build the subtree of an element. + + .. method:: start(tag, attrib) + + Opens a new element and returns it, without children and text. + + .. method:: data(data) + + Adds text to the current element, + or to the tail of the last closed one. + + .. method:: end(tag) + + Closes the current element and returns the same object + which was returned by :meth:`start`, with its text, + but still without children. + + .. method:: comment(text) + + Handles a comment and returns a comment element. + + .. method:: pi(target, text=None) + + Handles a processing instruction + and returns a processing instruction element. + + .. method:: close() + + Returns ``None``. + + .. versionadded:: next + + .. _elementtree-treebuilder-objects: TreeBuilder Objects @@ -1588,6 +1650,23 @@ XMLPullParser Objects Any events not yet retrieved when the parser is closed can still be read with :meth:`read_events`. + .. method:: expand(element) + + Build the subtree of *element* from the events which are already read + from the parser, and return *element*. + The events of its descendants are consumed and the corresponding + objects are added to it, up to the ``"end"`` event of *element*. + + Both the ``"start"`` and the ``"end"`` events should be reported, + and the ``"start"`` event of *element* should be already read. + :exc:`ValueError` is raised if the element is not complete yet; + feed more data to the parser and call :meth:`!expand` again. + The iterator returned by :func:`iterparse` has a method with the same + name which reads more data itself. + + .. versionadded:: next + + .. method:: read_events() Return an iterator over the events which have been encountered in the diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 2eccc8e35605596..c3d5c2458098bfe 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -715,6 +715,14 @@ xml building a tree for it. (Contributed by Serhiy Storchaka in :gh:`63102`.) +* Add :class:`~xml.etree.ElementTree.XMLPullTarget`, a parser target which + reports elements without children, and + :meth:`~xml.etree.ElementTree.XMLPullParser.expand`, which builds the + subtree of a reported element. + They allow to process a document of any size with a constant amount of + memory, like :mod:`xml.dom.pulldom`, but much faster. + (Contributed by Serhiy Storchaka in :gh:`63102`.) + zipfile ------- diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index f87a47045dd1713..2062bbabeec4454 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1681,6 +1681,42 @@ def test_target(self): ]) self.assertIsNone(it.root) + def test_pull_target(self): + with open(SIMPLE_XMLFILE, 'rb') as f: + it = ET.iterparse(f, events=('start', 'end'), + target=ET.XMLPullTarget()) + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('start', 'root'), + ('start', 'element'), + ('end', 'element'), + ('start', 'element'), + ('end', 'element'), + ('start', 'empty-element'), + ('end', 'empty-element'), + ('end', 'root'), + ]) + self.assertIsNone(it.root) + + def test_pull_target_expand(self): + with open(SIMPLE_XMLFILE, 'rb') as f: + it = ET.iterparse(f, events=('start', 'end'), + target=ET.XMLPullTarget()) + action, root = next(it) + self.assertEqual((action, root.tag), ('start', 'root')) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('start', 'element')) + self.assertIs(it.expand(elem), elem) + self.assertEqual(ET.tostring(elem).rstrip(), + b'text') + # the events of the subtree are consumed + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('start', 'element'), + ('end', 'element'), + ('start', 'empty-element'), + ('end', 'empty-element'), + ('end', 'root'), + ]) + def test_parser_with_target(self): with open(SIMPLE_XMLFILE, 'rb') as f: parser = ET.XMLParser(target=self.Target()) @@ -1879,6 +1915,59 @@ def assert_event_tuples(self, parser, expected, max_events=None): list(islice(parser.read_events(), max_events)), expected) + def test_pull_target(self): + # the target reports elements without children + parser = ET.XMLPullParser(events=('start', 'end', 'comment', 'pi'), + target=ET.XMLPullTarget()) + self._feed(parser, + "t") + events = list(parser.read_events()) + self.assertEqual([(action, getattr(obj.tag, '__name__', obj.tag)) + for action, obj in events], + [('start', 'root'), ('comment', 'Comment'), + ('pi', 'ProcessingInstruction'), + ('start', 'a'), ('start', 'b'), ('end', 'b'), + ('end', 'a'), ('end', 'root')]) + root = events[0][1] + self.assertEqual(root.attrib, {'a': '1'}) + self.assertEqual(len(root), 0) + self.assertEqual(events[1][1].text, ' c ') + self.assertEqual(events[2][1].text, 'pitarget data') + a_start, a_end = events[3][1], events[-2][1] + self.assertIs(a_start, a_end) + self.assertEqual(a_end.text, 't') + self.assertEqual(len(a_end), 0) + self.assertIsNone(parser.close()) + + def test_pull_target_expand(self): + parser = ET.XMLPullParser(events=('start', 'end'), + target=ET.XMLPullTarget()) + self._feed(parser, "tdeep") + parser.close() + events = parser.read_events() + action, root = next(events) + self.assertEqual((action, root.tag), ('start', 'root')) + action, a = next(events) + self.assertEqual((action, a.tag), ('start', 'a')) + self.assertIs(parser.expand(a), a) + self.assertEqual(ET.tostring(a), b'tdeep') + # the events of the subtree are consumed, the rest is intact + self.assertEqual([(action, obj.tag) for action, obj in events], + [('start', 'c'), ('end', 'c'), ('end', 'root')]) + + def test_pull_target_expand_incomplete(self): + parser = ET.XMLPullParser(events=('start', 'end'), + target=ET.XMLPullTarget()) + self._feed(parser, "t") + events = parser.read_events() + next(events) + action, a = next(events) + with self.assertRaises(ValueError): + parser.expand(a) + self._feed(parser, "") + self.assertIs(parser.expand(a), a) + self.assertEqual(ET.tostring(a), b't') + def assert_event_tags(self, parser, expected, max_events=None): events = islice(parser.read_events(), max_events) self.assertEqual([(action, elem.tag) for action, elem in events], diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index 3b4bfa3bd483c2d..0d009dd481ba2c0 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -84,7 +84,7 @@ "tostring", "tostringlist", "TreeBuilder", "XML", "XMLID", - "XMLParser", "XMLPullParser", + "XMLParser", "XMLPullParser", "XMLPullTarget", "register_namespace", "canonicalize", "C14NWriterTarget", ] @@ -1287,6 +1287,17 @@ def iterator(source): class IterParseIterator(collections.abc.Iterator): __next__ = gen.__next__ + def expand(self, element): + """Build the subtree of *element*, reading more data if needed.""" + while True: + try: + return pullparser.expand(element) + except ValueError: + data = source.read(16 * 1024) + if not data: + raise + pullparser.feed(data) + def close(self): nonlocal close_source if close_source: @@ -1365,6 +1376,39 @@ def read_events(self): else: yield event + def expand(self, element): + """Build the subtree of *element* from the queued events. + + The events of the descendants of *element* are consumed and the + corresponding objects are added to it, up to the "end" event of + *element* itself. Returns *element*. + + Both the "start" and the "end" events should be reported, and + the "start" event of *element* should be already read. + Raise ValueError if the element is not complete yet: feed more data + and call expand() again. + """ + events = self._events_queue + stack = [element] + n = 0 + for event in events: + n += 1 + if isinstance(event, Exception): + raise event + kind, obj = event + if kind == 'start': + stack[-1].append(obj) + stack.append(obj) + elif kind == 'end': + if obj is element: + for _ in range(n): + events.popleft() + return element + stack.pop() + elif kind in ('comment', 'pi'): + stack[-1].append(obj) + raise ValueError("the element is not complete yet") + def flush(self): if self._parser is None: raise ValueError("flush() called after end of stream") @@ -1551,6 +1595,82 @@ def _handle_single(self, factory, insert, *args): return elem +class XMLPullTarget: + """Parser target which reports elements without building a tree. + + It can be used with XMLPullParser and iterparse(). The reported object + is an Element with the tag, the attributes and, for the "end" event, the + text, but without children: they are reported as their own events. + Nothing is kept, so a document of any size can be parsed with a constant + amount of memory. + + Use XMLPullParser.expand() to build the subtree of an element. + + """ + def __init__(self, element_factory=None): + if element_factory is None: + element_factory = Element + self._factory = element_factory + self._stack = [] + self._last = None + self._tail = False + + def start(self, tag, attrib): + """Open a new element. + + Returns the new Element, without children and text. + """ + elem = self._factory(tag, attrib) + self._stack.append(elem) + self._last = elem + self._tail = False + return elem + + def data(self, data): + """Add text to the current element or to the last closed one.""" + last = self._last + if last is None: + return + if self._tail: + last.tail = last.tail + data if last.tail else data + else: + last.text = last.text + data if last.text else data + + def end(self, tag): + """Close the current element. + + Returns the same Element which was returned by start(), with its + text, but still without children. + """ + elem = self._stack.pop() + self._last = elem + self._tail = True + return elem + + def comment(self, text): + """Handle a comment. Returns a comment element.""" + elem = Comment(text) + self._last = elem + self._tail = True + return elem + + def pi(self, target, text=None): + """Handle a processing instruction. + + Returns a processing instruction element. + """ + elem = ProcessingInstruction(target, text) + self._last = elem + self._tail = True + return elem + + def close(self): + """Flush the buffers and return None.""" + self._stack.clear() + self._last = None + return None + + # also see ElementTree and TreeBuilder class XMLParser: """Element structure builder for XML source data based on the expat parser. diff --git a/Misc/NEWS.d/next/Library/2026-09-04-16-40-00.gh-issue-63102.Bt5nQ4.rst b/Misc/NEWS.d/next/Library/2026-09-04-16-40-00.gh-issue-63102.Bt5nQ4.rst new file mode 100644 index 000000000000000..ea7890aa7466ef8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-04-16-40-00.gh-issue-63102.Bt5nQ4.rst @@ -0,0 +1,6 @@ +Add :class:`xml.etree.ElementTree.XMLPullTarget`, a parser target which +reports elements without building a tree, and +:meth:`~xml.etree.ElementTree.XMLPullParser.expand`, which builds the subtree +of a reported element from the events which follow it. Together they allow +processing a document of any size with a constant amount of memory, like +:mod:`xml.dom.pulldom`, but much faster.