diff --git a/docs/testing/parity-testing/README.md b/docs/testing/parity-testing/README.md index d593676c41605..9127dc5794b45 100644 --- a/docs/testing/parity-testing/README.md +++ b/docs/testing/parity-testing/README.md @@ -248,3 +248,12 @@ localstack.testing.snapshots.transformer: Replacing regex '000000000000' with '1 localstack.testing.snapshots.transformer: Replacing regex 'us-east-1' with '' localstack.testing.snapshots.transformer: Replacing '1ad533b5-ac54-4354-a273-3ea885f0d59d' in snapshot with '' ``` + +### Test duration recording + +When a test runs successfully against AWS, its last validation date and duration are recorded in a corresponding ***.validation.json** file. +The validation date is recorded precisely, while test durations can vary between runs. +For example, test setup time may differ depending on whether a test runs in isolation or as part of a class test suite with class-level fixtures. +The recorded durations should be treated as approximate indicators of test execution time rather than precise measurements. +The goal of duration recording is to give _an idea_ about execution times. +If no duration is present in the validation file, it means the test has not been re-validated against AWS since duration recording was implemented. diff --git a/localstack-core/localstack/testing/pytest/validation_tracking.py b/localstack-core/localstack/testing/pytest/validation_tracking.py index ca8679fc4f1ac..cb3fd9eb48dae 100644 --- a/localstack-core/localstack/testing/pytest/validation_tracking.py +++ b/localstack-core/localstack/testing/pytest/validation_tracking.py @@ -9,17 +9,28 @@ import json import os from pathlib import Path -from typing import Optional +from typing import Dict, Optional -import pluggy import pytest +from pluggy import Result +from pytest import StashKey, TestReport from localstack.testing.aws.util import is_aws_cloud +durations_key = StashKey[Dict[str, float]]() +""" +Stores phase durations on the test node between execution phases. +See https://docs.pytest.org/en/latest/reference/reference.html#pytest.Stash +""" +test_failed_key = StashKey[bool]() +""" +Stores information from call execution phase about whether the test failed. +""" + -def find_snapshot_for_item(item: pytest.Item) -> Optional[dict]: +def find_validation_data_for_item(item: pytest.Item) -> Optional[dict]: base_path = os.path.join(item.fspath.dirname, item.fspath.purebasename) - snapshot_path = f"{base_path}.snapshot.json" + snapshot_path = f"{base_path}.validation.json" if not os.path.exists(snapshot_path): return None @@ -29,19 +40,45 @@ def find_snapshot_for_item(item: pytest.Item) -> Optional[dict]: return file_content.get(item.nodeid) -def find_validation_data_for_item(item: pytest.Item) -> Optional[dict]: - base_path = os.path.join(item.fspath.dirname, item.fspath.purebasename) - snapshot_path = f"{base_path}.validation.json" +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo): + """ + This hook is called after each test execution phase (setup, call, teardown). + """ + result: Result = yield + report: TestReport = result.get_result() - if not os.path.exists(snapshot_path): - return None + if call.when == "setup": + _makereport_setup(item, call) + elif call.when == "call": + _makereport_call(item, call) + elif call.when == "teardown": + _makereport_teardown(item, call) - with open(snapshot_path, "r") as fd: - file_content = json.load(fd) - return file_content.get(item.nodeid) + return report -def record_passed_validation(item: pytest.Item, timestamp: Optional[datetime.datetime] = None): +def _stash_phase_duration(call, item): + durations_by_phase = item.stash.setdefault(durations_key, {}) + durations_by_phase[call.when] = round(call.duration, 2) + + +def _makereport_setup(item: pytest.Item, call: pytest.CallInfo): + _stash_phase_duration(call, item) + + +def _makereport_call(item: pytest.Item, call: pytest.CallInfo): + _stash_phase_duration(call, item) + item.stash[test_failed_key] = call.excinfo is not None + + +def _makereport_teardown(item: pytest.Item, call: pytest.CallInfo): + _stash_phase_duration(call, item) + + # only update the file when running against AWS and the test finishes successfully + if not is_aws_cloud() or item.stash.get(test_failed_key, True): + return + base_path = os.path.join(item.fspath.dirname, item.fspath.purebasename) file_path = Path(f"{base_path}.validation.json") file_path.touch() @@ -49,45 +86,30 @@ def record_passed_validation(item: pytest.Item, timestamp: Optional[datetime.dat # read existing state from file try: content = json.load(fd) - except json.JSONDecodeError: # expected on first try (empty file) + except json.JSONDecodeError: # expected on the first try (empty file) content = {} - # update for this pytest node - if not timestamp: - timestamp = datetime.datetime.now(tz=datetime.timezone.utc) - content[item.nodeid] = {"last_validated_date": timestamp.isoformat(timespec="seconds")} + test_execution_data = content.setdefault(item.nodeid, {}) - # save updates - fd.seek(0) - json.dump(content, fd, indent=2, sort_keys=True) - fd.write("\n") # add trailing newline for linter and Git compliance + timestamp = datetime.datetime.now(tz=datetime.timezone.utc) + test_execution_data["last_validated_date"] = timestamp.isoformat(timespec="seconds") + durations_by_phase = item.stash[durations_key] + test_execution_data["durations_in_seconds"] = durations_by_phase -# TODO: we should skip if we're updating snapshots -# make sure this is *AFTER* snapshot comparison => tryfirst=True -@pytest.hookimpl(hookwrapper=True, tryfirst=True) -def pytest_runtest_call(item: pytest.Item): - outcome: pluggy.Result = yield + total_duration = sum(durations_by_phase.values()) + durations_by_phase["total"] = round(total_duration, 2) - # we only want to track passed runs against AWS - if not is_aws_cloud() or outcome.excinfo: - return + # For json.dump sorted test entries enable consistent diffs. + # But test execution data is more readable in insert order for each step (setup, call, teardown). + # Hence, not using global sort_keys=True for json.dump but rather additionally sorting top-level dict only. + content = dict(sorted(content.items())) - record_passed_validation(item) - - -# this is a sort of utility used for retroactively creating validation files in accordance with existing snapshot files -# it takes the recorded date from a snapshot and sets it to the last validated date -# @pytest.hookimpl(trylast=True) -# def pytest_collection_modifyitems(session: pytest.Session, config: pytest.Config, items: list[pytest.Item]): -# for item in items: -# snapshot_entry = find_snapshot_for_item(item) -# if not snapshot_entry: -# continue -# -# snapshot_update_timestamp = datetime.datetime.strptime(snapshot_entry["recorded-date"], "%d-%m-%Y, %H:%M:%S").astimezone(tz=datetime.timezone.utc) -# -# record_passed_validation(item, snapshot_update_timestamp) + # save updates + fd.truncate(0) # clear existing content + fd.seek(0) + json.dump(content, fd, indent=2) + fd.write("\n") # add trailing newline for linter and Git compliance @pytest.hookimpl diff --git a/tests/aws/services/lambda_/event_source_mapping/test_cfn_resource.snapshot.json b/tests/aws/services/lambda_/event_source_mapping/test_cfn_resource.snapshot.json index 1cc37cf30ca33..618589334f8f8 100644 --- a/tests/aws/services/lambda_/event_source_mapping/test_cfn_resource.snapshot.json +++ b/tests/aws/services/lambda_/event_source_mapping/test_cfn_resource.snapshot.json @@ -1,6 +1,6 @@ { "tests/aws/services/lambda_/event_source_mapping/test_cfn_resource.py::test_adding_tags": { - "recorded-date": "06-11-2024, 11:55:29", + "recorded-date": "19-05-2025, 09:32:18", "recorded-content": { "event-source-mapping-tags": { "Tags": { diff --git a/tests/aws/services/lambda_/event_source_mapping/test_cfn_resource.validation.json b/tests/aws/services/lambda_/event_source_mapping/test_cfn_resource.validation.json index 7bbb9723d78fe..2a6ef1af1c4db 100644 --- a/tests/aws/services/lambda_/event_source_mapping/test_cfn_resource.validation.json +++ b/tests/aws/services/lambda_/event_source_mapping/test_cfn_resource.validation.json @@ -1,5 +1,11 @@ { "tests/aws/services/lambda_/event_source_mapping/test_cfn_resource.py::test_adding_tags": { - "last_validated_date": "2024-11-06T11:55:29+00:00" + "last_validated_date": "2025-05-19T09:33:12+00:00", + "durations_in_seconds": { + "setup": 0.54, + "call": 69.88, + "teardown": 54.76, + "total": 125.18 + } } }