localstack · tiurin · May 29, 2025 · May 15, 2025 · May 15, 2025 · May 15, 2025
@@ -248,3 +248,12 @@ localstack.testing.snapshots.transformer: Replacing regex '000000000000' with '1
 localstack.testing.snapshots.transformer: Replacing regex 'us-east-1' with '<region>'
 localstack.testing.snapshots.transformer: Replacing '1ad533b5-ac54-4354-a273-3ea885f0d59d' in snapshot with '<uuid:1>'
 ```
+
+### Test duration recording
+
+When a test runs successfully against AWS, its last validation date and duration are recorded in a corresponding ***.validation.json** file.
+The validation date is recorded precisely, while test durations can vary between runs.
+For example, test setup time may differ depending on whether a test runs in isolation or as part of a class test suite with class-level fixtures.
+The recorded durations should be treated as approximate indicators of test execution time rather than precise measurements.
+The goal of duration recording is to give _an idea_ about execution times.
+If no duration is present in the validation file, it means the test has not been re-validated against AWS since duration recording was implemented.
@@ -9,17 +9,28 @@
 import json
 import os
 from pathlib import Path
-from typing import Optional
+from typing import Dict, Optional
 
-import pluggy
 import pytest
+from pluggy import Result
+from pytest import StashKey, TestReport
 
 from localstack.testing.aws.util import is_aws_cloud
 
+durations_key = StashKey[Dict[str, float]]()
+"""
+Stores phase durations on the test node between execution phases.
+See https://docs.pytest.org/en/latest/reference/reference.html#pytest.Stash
+"""
+test_failed_key = StashKey[bool]()
+"""
+Stores information from call execution phase about whether the test failed.
+"""
+
 
-def find_snapshot_for_item(item: pytest.Item) -> Optional[dict]:
+def find_validation_data_for_item(item: pytest.Item) -> Optional[dict]:
     base_path = os.path.join(item.fspath.dirname, item.fspath.purebasename)
-    snapshot_path = f"{base_path}.snapshot.json"
+    snapshot_path = f"{base_path}.validation.json"
 
     if not os.path.exists(snapshot_path):
         return None
@@ -29,65 +40,76 @@ def find_snapshot_for_item(item: pytest.Item) -> Optional[dict]:
         return file_content.get(item.nodeid)
 
 
-def find_validation_data_for_item(item: pytest.Item) -> Optional[dict]:
-    base_path = os.path.join(item.fspath.dirname, item.fspath.purebasename)
-    snapshot_path = f"{base_path}.validation.json"
+@pytest.hookimpl(hookwrapper=True)
+def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo):
+    """
+    This hook is called after each test execution phase (setup, call, teardown).
+    """
+    result: Result = yield
+    report: TestReport = result.get_result()
 
-    if not os.path.exists(snapshot_path):
-        return None
+    if call.when == "setup":
+        _makereport_setup(item, call)
+    elif call.when == "call":
+        _makereport_call(item, call)
+    elif call.when == "teardown":
+        _makereport_teardown(item, call)
 
-    with open(snapshot_path, "r") as fd:
-        file_content = json.load(fd)
-        return file_content.get(item.nodeid)
+    return report
 
 
-def record_passed_validation(item: pytest.Item, timestamp: Optional[datetime.datetime] = None):
+def _stash_phase_duration(call, item):
+    durations_by_phase = item.stash.setdefault(durations_key, {})
+    durations_by_phase[call.when] = round(call.duration, 2)
+
+
+def _makereport_setup(item: pytest.Item, call: pytest.CallInfo):
+    _stash_phase_duration(call, item)
+
+
+def _makereport_call(item: pytest.Item, call: pytest.CallInfo):
+    _stash_phase_duration(call, item)
+    item.stash[test_failed_key] = call.excinfo is not None
+
+
+def _makereport_teardown(item: pytest.Item, call: pytest.CallInfo):
+    _stash_phase_duration(call, item)
+
+    # only update the file when running against AWS and the test finishes successfully
+    if not is_aws_cloud() or item.stash.get(test_failed_key, True):
+        return
+
     base_path = os.path.join(item.fspath.dirname, item.fspath.purebasename)
     file_path = Path(f"{base_path}.validation.json")
     file_path.touch()
     with file_path.open(mode="r+") as fd:
         # read existing state from file
         try:
             content = json.load(fd)
-        except json.JSONDecodeError:  # expected on first try (empty file)
+        except json.JSONDecodeError:  # expected on the first try (empty file)
             content = {}
 
-        # update for this pytest node
-        if not timestamp:
-            timestamp = datetime.datetime.now(tz=datetime.timezone.utc)
-        content[item.nodeid] = {"last_validated_date": timestamp.isoformat(timespec="seconds")}
+        test_execution_data = content.setdefault(item.nodeid, {})
 
-        # save updates
-        fd.seek(0)
-        json.dump(content, fd, indent=2, sort_keys=True)
-        fd.write("\n")  # add trailing newline for linter and Git compliance
+        timestamp = datetime.datetime.now(tz=datetime.timezone.utc)
+        test_execution_data["last_validated_date"] = timestamp.isoformat(timespec="seconds")
 
+        durations_by_phase = item.stash[durations_key]
+        test_execution_data["durations_in_seconds"] = durations_by_phase
 
-# TODO: we should skip if we're updating snapshots
-# make sure this is *AFTER* snapshot comparison => tryfirst=True
-@pytest.hookimpl(hookwrapper=True, tryfirst=True)
-def pytest_runtest_call(item: pytest.Item):
-    outcome: pluggy.Result = yield
+        total_duration = sum(durations_by_phase.values())
+        durations_by_phase["total"] = round(total_duration, 2)
 
-    # we only want to track passed runs against AWS
-    if not is_aws_cloud() or outcome.excinfo:
-        return
+        # For json.dump sorted test entries enable consistent diffs.
+        # But test execution data is more readable in insert order for each step (setup, call, teardown).
+        # Hence, not using global sort_keys=True for json.dump but rather additionally sorting top-level dict only.
+        content = dict(sorted(content.items()))
 
-    record_passed_validation(item)
-
-
-# this is a sort of utility used for retroactively creating validation files in accordance with existing snapshot files
-# it takes the recorded date from a snapshot and sets it to the last validated date
-# @pytest.hookimpl(trylast=True)
-# def pytest_collection_modifyitems(session: pytest.Session, config: pytest.Config, items: list[pytest.Item]):
-#     for item in items:
-#         snapshot_entry = find_snapshot_for_item(item)
-#         if not snapshot_entry:
-#             continue
-#
-#         snapshot_update_timestamp = datetime.datetime.strptime(snapshot_entry["recorded-date"], "%d-%m-%Y, %H:%M:%S").astimezone(tz=datetime.timezone.utc)
-#
-#         record_passed_validation(item, snapshot_update_timestamp)
+        # save updates
+        fd.truncate(0)  # clear existing content
+        fd.seek(0)
+        json.dump(content, fd, indent=2)
+        fd.write("\n")  # add trailing newline for linter and Git compliance
 
 
 @pytest.hookimpl

@@ -1,6 +1,6 @@
 {
   "tests/aws/services/lambda_/event_source_mapping/test_cfn_resource.py::test_adding_tags": {
-    "recorded-date": "06-11-2024, 11:55:29",
+    "recorded-date": "19-05-2025, 09:32:18",
     "recorded-content": {
       "event-source-mapping-tags": {
         "Tags": {

@@ -1,5 +1,11 @@
 {
   "tests/aws/services/lambda_/event_source_mapping/test_cfn_resource.py::test_adding_tags": {
-    "last_validated_date": "2024-11-06T11:55:29+00:00"
+    "last_validated_date": "2025-05-19T09:33:12+00:00",
+    "durations_in_seconds": {
+      "setup": 0.54,
+      "call": 69.88,
+      "teardown": 54.76,
+      "total": 125.18
+    }
   }
 }