From 267b37cd47a7e9dd0b988eef9543a0ba13bcb614 Mon Sep 17 00:00:00 2001 From: Martin Vergier Date: Mon, 22 Jun 2026 16:04:00 +0000 Subject: [PATCH 1/4] feat: replace int return of insert_into_target_tables with LoadStats dataclass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit insert_into_target_tables now returns LoadStats(inserted, existing, duration_temp_insert, duration_merge, duration_cleanup). merge_into_target_tables returns MergeStats(inserted, existing, duration). insert_into_temp_tables returns the phase duration as a float. Row counts use the first INSERT rowcount per table (the data-table INSERT; join-table INSERTs are ignored). Backends that do not report rowcount for INSERT…FROM SELECT (e.g. DuckDB) silently yield 0 for inserted/existing. Both types are exported from the package root. Co-Authored-By: Claude Sonnet 4.6 --- src/xml2db/__init__.py | 4 +- src/xml2db/document.py | 108 ++++++++++++++++++++++++++++++++++------- 2 files changed, 94 insertions(+), 18 deletions(-) diff --git a/src/xml2db/__init__.py b/src/xml2db/__init__.py index ee76143..07bd4ad 100644 --- a/src/xml2db/__init__.py +++ b/src/xml2db/__init__.py @@ -1,5 +1,5 @@ from .model import DataModel -from .document import Document +from .document import Document, LoadStats, MergeStats from .table import ( DataModelTable, DataModelTableReused, @@ -12,6 +12,8 @@ __all__ = [ "DataModel", "Document", + "LoadStats", + "MergeStats", "DataModelTable", "DataModelTableReused", "DataModelTableDuplicated", diff --git a/src/xml2db/document.py b/src/xml2db/document.py index 6aa08da..eb87da1 100644 --- a/src/xml2db/document.py +++ b/src/xml2db/document.py @@ -1,10 +1,12 @@ import csv import datetime import logging +import time +from dataclasses import dataclass from io import BytesIO from typing import Union, TYPE_CHECKING from zoneinfo import ZoneInfo -from sqlalchemy import Column, Table, text, select +from sqlalchemy import Column, Table, func, text, select from sqlalchemy.engine import Connection from sqlalchemy.sql.expression import TextClause from lxml import etree @@ -17,6 +19,50 @@ logger = logging.getLogger(__name__) +@dataclass +class MergeStats: + """Statistics returned by :meth:`~xml2db.document.Document.merge_into_target_tables`. + + Attributes: + inserted: Rows from the document that were written to target tables. + May be 0 on backends that do not report rowcount for ``INSERT … FROM SELECT`` + (e.g. DuckDB). + existing: Rows from the document that were not written because an identical record + was already present (hash-deduplicated for reused tables, or because a parent + record was already existing for duplicated tables). + May be 0 on backends that do not report rowcount for ``INSERT … FROM SELECT``. + duration: Seconds spent executing merge statements. + """ + + inserted: int + existing: int + duration: float + + +@dataclass +class LoadStats: + """Statistics returned by :meth:`~xml2db.document.Document.insert_into_target_tables`. + + Attributes: + inserted: Rows from the document that were written to target tables. + May be 0 on backends that do not report rowcount for ``INSERT … FROM SELECT`` + (e.g. DuckDB). + existing: Rows from the document that were not written because an identical record + was already present (hash-deduplicated for reused tables, or because a parent + record was already existing for duplicated tables). + May be 0 on backends that do not report rowcount for ``INSERT … FROM SELECT``. + duration_temp_insert: Seconds spent inserting data into temporary staging tables. + duration_merge: Seconds spent merging temporary tables into target tables. + duration_cleanup: Seconds spent dropping temporary tables. + """ + + inserted: int + existing: int + duration_temp_insert: float + duration_merge: float + duration_cleanup: float + + class Document: """A class to represent a single XML file with its data, based on a given XSD. @@ -373,7 +419,7 @@ def insert_into_temp_tables( max_lines: int = -1, bulk_load: bool | None = None, bulk_load_threshold: int | None = None, - ) -> None: + ) -> float: """Insert data into temporary tables (Re)creates temp tables before inserting data. @@ -385,7 +431,11 @@ def insert_into_temp_tables( use bulk loading when available and fall back silently. bulk_load_threshold: Minimum number of records to trigger bulk loading. ``None`` delegates the choice to the dialect. + + Returns: + Seconds spent on this phase. """ + t0 = time.perf_counter() logger.info(f"Dropping temp tables if exist for {self.xml_file_path}") self.model.drop_all_temp_tables() @@ -410,8 +460,9 @@ def insert_into_temp_tables( bulk_load_threshold=bulk_load_threshold, ) start_idx = start_idx + batch_size + return time.perf_counter() - t0 - def merge_into_target_tables(self, single_transaction: bool = True) -> int: + def merge_into_target_tables(self, single_transaction: bool = True) -> MergeStats: """Merge data into target data model Execute all update and insert statements needed to merge temporary tables content into target tables. @@ -421,9 +472,11 @@ def merge_into_target_tables(self, single_transaction: bool = True) -> int: scope required to ensure database consistency? Returns: - The number of inserted rows + A :class:`MergeStats` object with inserted/existing row counts and phase duration. """ - inserted_rows_count = 0 + inserted = 0 + existing = 0 + t0 = time.perf_counter() for tables in ( [self.model.fk_ordered_tables] if single_transaction @@ -431,16 +484,27 @@ def merge_into_target_tables(self, single_transaction: bool = True) -> int: ): with self.model.engine.begin() as conn: for tb in tables: + # Within each table's statement stream the first INSERT is always the + # main data-table insert; subsequent INSERTs belong to n-n join tables. + table_inserted = None for query in tb.get_merge_temp_records_statements(): result = conn.execute(query) - if query.is_insert: - inserted_rows_count += result.rowcount - if inserted_rows_count == 0: + if query.is_insert and table_inserted is None: + # rowcount is -1 on backends that do not report it for + # INSERT … FROM SELECT (e.g. DuckDB); skip those tables. + if result.rowcount >= 0: + table_inserted = result.rowcount + if table_inserted is not None: + inserted += table_inserted + if tb.is_reused and tb.type_name in self.data: + existing += ( + len(self.data[tb.type_name]["records"]) - table_inserted + ) + if inserted == 0: logger.info("No rows were inserted!") else: - logger.info(f"Inserted rows: {inserted_rows_count}") - - return inserted_rows_count + logger.info(f"Inserted rows: {inserted}, existing rows: {existing}") + return MergeStats(inserted=inserted, existing=existing, duration=time.perf_counter() - t0) def insert_into_target_tables( self, @@ -448,7 +512,7 @@ def insert_into_target_tables( max_lines: int = -1, bulk_load: bool | None = None, bulk_load_threshold: int | None = None, - ) -> int: + ) -> LoadStats: """Insert and merge data into the database Insert data into temporary tables and then merge temporary tables into target tables. @@ -465,8 +529,10 @@ def insert_into_target_tables( loading. ``None`` delegates the choice to the dialect. Returns: - The number of inserted rows + A :class:`LoadStats` object with inserted/existing row counts and per-phase durations. """ + merge_stats = None + duration_cleanup = 0.0 try: self.model.create_db_schema() except Exception as e: @@ -476,7 +542,7 @@ def insert_into_target_tables( logger.error(e) raise try: - self.insert_into_temp_tables(max_lines, bulk_load, bulk_load_threshold) + duration_temp = self.insert_into_temp_tables(max_lines, bulk_load, bulk_load_threshold) except Exception as e: logger.error( f"Error while importing into temporary tables from {self.xml_file_path}" @@ -489,7 +555,7 @@ def insert_into_target_tables( ) try: self.model.create_all_tables() # Create target tables if not exist - inserted_rows = self.merge_into_target_tables(single_transaction) + merge_stats = self.merge_into_target_tables(single_transaction) except Exception as e: logger.error( f"Error while merging temporary tables into target tables for {self.xml_file_path}" @@ -498,9 +564,17 @@ def insert_into_target_tables( raise finally: logger.info(f"Dropping temporary tables for {self.xml_file_path}") + t0 = time.perf_counter() self.model.drop_all_temp_tables() - - return inserted_rows + duration_cleanup = time.perf_counter() - t0 + + return LoadStats( + inserted=merge_stats.inserted, + existing=merge_stats.existing, + duration_temp_insert=duration_temp, + duration_merge=merge_stats.duration, + duration_cleanup=duration_cleanup, + ) def extract_from_database( self, From ea1d6cfe0fd242f70968bf779b5b4fc1d430ca51 Mon Sep 17 00:00:00 2001 From: Martin Vergier Date: Mon, 22 Jun 2026 16:06:23 +0000 Subject: [PATCH 2/4] docs: add LoadStats and MergeStats to API reference and overview --- docs/api/document.md | 6 ++++++ docs/api/overview.md | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/api/document.md b/docs/api/document.md index aef5567..0490d6f 100644 --- a/docs/api/document.md +++ b/docs/api/document.md @@ -6,3 +6,9 @@ description: "API reference for the xml2db Document class: methods for inserting # Document ::: xml2db.document.Document + +## Load statistics + +::: xml2db.document.LoadStats + +::: xml2db.document.MergeStats diff --git a/docs/api/overview.md b/docs/api/overview.md index cfd15d9..064b7ab 100644 --- a/docs/api/overview.md +++ b/docs/api/overview.md @@ -28,7 +28,8 @@ description: "Overview of the xml2db Python API: building DataModel objects, par * [`DataModel.parse_xml`](data_model.md/#xml2db.model.DataModel.parse_xml): read and parse a XML document, which is loaded in memory * [`Document.insert_into_target_tables`](document.md/#xml2db.document.Document.insert_into_target_tables): load a file - into the database + into the database; returns a [`LoadStats`](document.md/#xml2db.document.LoadStats) object with inserted/existing + row counts and per-phase durations ## *Advanced use:* loading data into the database From 855c7920a786027be16539cb09a961540d61c215 Mon Sep 17 00:00:00 2001 From: Martin Vergier Date: Mon, 22 Jun 2026 16:45:57 +0000 Subject: [PATCH 3/4] test: verify LoadStats return value from insert_into_target_tables --- tests/test_roundtrip.py | 50 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/tests/test_roundtrip.py b/tests/test_roundtrip.py index 8f63496..a87c5a3 100644 --- a/tests/test_roundtrip.py +++ b/tests/test_roundtrip.py @@ -3,11 +3,59 @@ import pytest from lxml import etree +from xml2db import LoadStats, MergeStats from xml2db.xml_converter import XMLConverter, remove_record_hash -from .conftest import list_xml_path +from .conftest import list_xml_path, models_path from .sample_models import models +@pytest.mark.dbtest +def test_load_stats(conn_string): + """Test that insert_into_target_tables returns a LoadStats with correct structure""" + from xml2db import DataModel + import sqlalchemy + + model = DataModel( + str(os.path.join(models_path, "orders", "orders.xsd")), + connection_string=conn_string, + db_schema="test_xml2db_stats", + model_config={ + "metadata_columns": [{"name": "src", "type": sqlalchemy.String(256)}] + }, + ) + model.create_db_schema() + model.drop_all_tables() + model.create_all_tables() + xml_path = str(os.path.join(models_path, "orders", "xml", "order1.xml")) + try: + # first load + doc = model.parse_xml(xml_path, metadata={"src": "a"}) + stats = doc.insert_into_target_tables() + + assert isinstance(stats, LoadStats) + assert stats.inserted >= 0 + assert stats.existing == 0 # nothing pre-existing on first load + assert stats.duration_temp_insert > 0 + assert stats.duration_merge > 0 + assert stats.duration_cleanup > 0 + + # second load of same file — reused rows should be existing (backends that + # report rowcount); on DuckDB both will be 0, which is also acceptable + doc2 = model.parse_xml(xml_path, metadata={"src": "b"}) + stats2 = doc2.insert_into_target_tables() + + assert isinstance(stats2, LoadStats) + assert stats2.inserted >= 0 + assert stats2.existing >= 0 + assert stats2.inserted + stats2.existing >= 0 + # where rowcount is supported: all reused records from the first load + # are now existing; inserted should be 0 or very small + if stats.inserted > 0: + assert stats2.existing > 0 + finally: + model.drop_all_tables() + + @pytest.mark.dbtest @pytest.mark.parametrize( "model_config", From 5da44db893197541ea9bcf97bbdfc326175ddfd0 Mon Sep 17 00:00:00 2001 From: Martin Vergier Date: Mon, 22 Jun 2026 16:48:46 +0000 Subject: [PATCH 4/4] fix: use spawn context in test_multiprocessing to suppress Python 3.12 fork deprecation warning --- tests/test_multiprocessing.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_multiprocessing.py b/tests/test_multiprocessing.py index 674de73..7d4605c 100644 --- a/tests/test_multiprocessing.py +++ b/tests/test_multiprocessing.py @@ -65,10 +65,11 @@ def test_multiprocessing_file_duckdb(): """ with tempfile.TemporaryDirectory() as tmpdir: db_path = os.path.join(tmpdir, "test.duckdb") - lock = multiprocessing.Lock() + ctx = multiprocessing.get_context("spawn") + lock = ctx.Lock() processes = [ - multiprocessing.Process( + ctx.Process( target=_load_xml_file, args=(xml_path, _XSD, db_path, lock), )