-
- | Test Name |
- Baseline image |
- Diff |
- New image |
-
-"""
-
def _download_file(baseline, filename):
# Note that baseline can be a comma-separated list of URLs that we can
@@ -162,7 +133,7 @@ def pytest_addoption(parser):
group.addoption('--mpl-generate-summary', action='store',
help="Generate a summary report of any failed tests"
", in --mpl-results-path. The type of the report should be "
- "specified. Supported types are `html` and `json`. "
+ "specified. Supported types are `html`, `json` and `basic-html`. "
"Multiple types can be specified separated by commas.")
results_path_help = "directory for test results, relative to location where py.test is run"
@@ -712,105 +683,6 @@ def item_function_wrapper(*args, **kwargs):
else:
item.obj = item_function_wrapper
- def generate_stats(self):
- """
- Generate a dictionary of summary statistics.
- """
- stats = {'passed': 0, 'failed': 0, 'passed_baseline': 0, 'failed_baseline': 0, 'skipped': 0}
- for test in self._test_results.values():
- if test['status'] == 'passed':
- stats['passed'] += 1
- if test['rms'] is not None:
- stats['failed_baseline'] += 1
- elif test['status'] == 'failed':
- stats['failed'] += 1
- if test['rms'] is None:
- stats['passed_baseline'] += 1
- elif test['status'] == 'skipped':
- stats['skipped'] += 1
- else:
- raise ValueError(f"Unknown test status '{test['status']}'.")
- self._test_stats = stats
-
- def generate_summary_html(self):
- """
- Generate a simple HTML table of the failed test results
- """
- html_file = self.results_dir / 'fig_comparison.html'
- with open(html_file, 'w') as f:
-
- passed = f"{self._test_stats['passed']} passed"
- if self._test_stats['failed_baseline'] > 0:
- passed += (" hash comparison, although "
- f"{self._test_stats['failed_baseline']} "
- "of those have a different baseline image")
-
- failed = f"{self._test_stats['failed']} failed"
- if self._test_stats['passed_baseline'] > 0:
- failed += (" hash comparison, although "
- f"{self._test_stats['passed_baseline']} "
- "of those have a matching baseline image")
-
- f.write(HTML_INTRO.replace('%summary%', f'{passed}.
{failed}.
'))
-
- for test_name in sorted(self._test_results.keys()):
- summary = self._test_results[test_name]
-
- if not self.results_always and summary['result_image'] is None:
- continue # Don't show test if no result image
-
- if summary['rms'] is None and summary['tolerance'] is not None:
- rms = (f'\n'
- f' RMS: '
- f' < {summary["tolerance"]}\n'
- f'
')
- elif summary['rms'] is not None:
- rms = (f'\n'
- f' RMS: '
- f' {summary["rms"]}\n'
- f'
')
- else:
- rms = ''
-
- hashes = ''
- if summary['baseline_hash'] is not None:
- hashes += (f' Baseline: '
- f'{summary["baseline_hash"]}
\n')
- if summary['result_hash'] is not None:
- hashes += (f' Result: '
- f'{summary["result_hash"]}
\n')
- if len(hashes) > 0:
- if summary["baseline_hash"] == summary["result_hash"]:
- hash_result = 'passed'
- else:
- hash_result = 'failed'
- hashes = f'\n{hashes}
'
-
- images = {}
- for image_type in ['baseline_image', 'diff_image', 'result_image']:
- if summary[image_type] is not None:
- images[image_type] = f'
'
- else:
- images[image_type] = ''
-
- f.write(f'\n'
- ' \n'
- ' \n'
- f' {test_name} \n'
- f' {summary["status"]} \n'
- f' {rms}{hashes}\n'
- ' | \n'
- f' {images["baseline_image"]} | \n'
- f' {images["diff_image"]} | \n'
- f' {images["result_image"]} | \n'
- '
\n\n')
-
- f.write('
\n')
- f.write('\n')
- f.write('')
-
- return html_file
-
def generate_summary_json(self):
json_file = self.results_dir / 'results.json'
with open(json_file, 'w') as f:
@@ -843,13 +715,14 @@ def pytest_unconfigure(self, config):
if self._test_results[test_name][image_type] == '%EXISTS%':
self._test_results[test_name][image_type] = str(directory / filename)
- self.generate_stats()
-
if 'json' in self.generate_summary:
summary = self.generate_summary_json()
print(f"A JSON report can be found at: {summary}")
if 'html' in self.generate_summary:
- summary = self.generate_summary_html()
+ summary = generate_summary_html(self._test_results, self.results_dir)
+ print(f"A summary of the failed tests can be found at: {summary}")
+ if 'basic-html' in self.generate_summary:
+ summary = generate_summary_basic_html(self._test_results, self.results_dir)
print(f"A summary of the failed tests can be found at: {summary}")
diff --git a/pytest_mpl/summary/__init__.py b/pytest_mpl/summary/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/pytest_mpl/summary/html.py b/pytest_mpl/summary/html.py
new file mode 100644
index 00000000..858cfe21
--- /dev/null
+++ b/pytest_mpl/summary/html.py
@@ -0,0 +1,288 @@
+import os
+import sys
+import shutil
+
+if sys.version_info >= (3, 8):
+ from functools import cached_property
+else:
+ cached_property = property
+
+from jinja2 import Environment, PackageLoader, select_autoescape
+
+__all__ = ['generate_summary_html', 'generate_summary_basic_html']
+
+
+class Results:
+ """
+ Data for the whole test run, used for providing data to the template.
+
+ Parameters
+ ----------
+ results : dict
+ The `pytest_mpl.plugin.ImageComparison._test_results` object.
+ title : str
+ Value for HTML