diff --git a/CHANGES.md b/CHANGES.md index 8e6c55e8..470ace47 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,10 @@ 0.15 (unreleased) ----------------- +- An updated hash library will be saved to the results directory when + generating a HTML summary page or when the `--mpl-results-always` flag is + set. A button to download this file is included in the HTML summary. + Various bugfixes, test improvements and documentation updates. [#138] 0.14 (2022-02-09) ----------------- diff --git a/README.rst b/README.rst index 4a80aec3..3dd9b742 100644 --- a/README.rst +++ b/README.rst @@ -86,6 +86,10 @@ can either be specified via the ``--mpl-hash-library=`` command line argument, or via the ``hash_library=`` keyword argument to the ``@pytest.mark.mpl_image_compare`` decorator. +When generating a hash library, the tests will also be run as usual against the +existing hash library specified by ``--mpl-hash-library`` or the keyword argument. +However, generating baseline images will always result in the tests being skipped. + Hybrid Mode: Hashes and Images ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -225,6 +229,12 @@ test (based on the hash library) also shown in the generated summary. This option is applied automatically when generating a HTML summary. +When the ``--mpl-results-always`` option is active, and some hash +comparison tests are performed, a hash library containing all the +result hashes will also be saved to the root of the results directory. +The filename will be extracted from ``--mpl-generate-hash-library``, +``--mpl-hash-library`` or ``hash_library=`` in that order. + Base style ^^^^^^^^^^ diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index d147e57c..3891dbc1 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -181,8 +181,6 @@ def pytest_configure(config): if generate_dir is not None: if baseline_dir is not None: warnings.warn("Ignoring --mpl-baseline-path since --mpl-generate-path is set") - if results_dir is not None and generate_dir is not None: - warnings.warn("Ignoring --mpl-result-path since --mpl-generate-path is set") if baseline_dir is not None and not baseline_dir.startswith(("https", "http")): baseline_dir = os.path.abspath(baseline_dir) @@ -283,6 +281,12 @@ def __init__(self, self.results_dir = Path(tempfile.mkdtemp(dir=self.results_dir)) self.results_dir.mkdir(parents=True, exist_ok=True) + # Decide what to call the downloadable results hash library + if self.hash_library is not None: + self.results_hash_library_name = self.hash_library.name + else: # Use the first filename encountered in a `hash_library=` kwarg + self.results_hash_library_name = None + # We need global state to store all the hashes generated over the run self._generated_hash_library = {} self._test_results = {} @@ -390,11 +394,14 @@ def generate_baseline_image(self, item, fig): if not os.path.exists(self.generate_dir): os.makedirs(self.generate_dir) - fig.savefig(str((self.generate_dir / self.generate_filename(item)).absolute()), - **savefig_kwargs) + baseline_filename = self.generate_filename(item) + baseline_path = (self.generate_dir / baseline_filename).absolute() + fig.savefig(str(baseline_path), **savefig_kwargs) close_mpl_figure(fig) + return baseline_path + def generate_image_hash(self, item, fig): """ For a `matplotlib.figure.Figure`, returns the SHA256 hash as a hexadecimal @@ -435,6 +442,7 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None): if not os.path.exists(baseline_image_ref): summary['status'] = 'failed' + summary['image_status'] = 'missing' error_message = ("Image file not found for comparison test in: \n\t" f"{self.get_baseline_directory(item)}\n" "(This is expected for new tests.)\n" @@ -456,6 +464,7 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None): actual_shape = imread(str(test_image)).shape[:2] if expected_shape != actual_shape: summary['status'] = 'failed' + summary['image_status'] = 'diff' error_message = SHAPE_MISMATCH_ERROR.format(expected_path=baseline_image, expected_shape=expected_shape, actual_path=test_image, @@ -467,10 +476,12 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None): summary['tolerance'] = tolerance if results is None: summary['status'] = 'passed' + summary['image_status'] = 'match' summary['status_msg'] = 'Image comparison passed.' return None else: summary['status'] = 'failed' + summary['image_status'] = 'diff' summary['rms'] = results['rms'] diff_image = (result_dir / 'result-failed-diff.png').absolute() summary['diff_image'] = diff_image.relative_to(self.results_dir).as_posix() @@ -496,6 +507,10 @@ def compare_image_to_hash_library(self, item, fig, result_dir, summary=None): compare = self.get_compare(item) savefig_kwargs = compare.kwargs.get('savefig_kwargs', {}) + if not self.results_hash_library_name: + # Use hash library name of current test as results hash library name + self.results_hash_library_name = Path(compare.kwargs.get("hash_library", "")).name + hash_library_filename = self.hash_library or compare.kwargs.get('hash_library', None) hash_library_filename = (Path(item.fspath).parent / hash_library_filename).absolute() @@ -512,14 +527,17 @@ def compare_image_to_hash_library(self, item, fig, result_dir, summary=None): if baseline_hash is None: # hash-missing summary['status'] = 'failed' + summary['hash_status'] = 'missing' summary['status_msg'] = (f"Hash for test '{hash_name}' not found in {hash_library_filename}. " f"Generated hash is {test_hash}.") elif test_hash == baseline_hash: # hash-match hash_comparison_pass = True summary['status'] = 'passed' + summary['hash_status'] = 'match' summary['status_msg'] = 'Test hash matches baseline hash.' else: # hash-diff summary['status'] = 'failed' + summary['hash_status'] = 'diff' summary['status_msg'] = (f"Hash {test_hash} doesn't match hash " f"{baseline_hash} in library " f"{hash_library_filename} for test {hash_name}.") @@ -544,7 +562,8 @@ def compare_image_to_hash_library(self, item, fig, result_dir, summary=None): except Exception as baseline_error: # Append to test error later baseline_comparison = str(baseline_error) else: # Update main summary - for k in ['baseline_image', 'diff_image', 'rms', 'tolerance', 'result_image']: + for k in ['image_status', 'baseline_image', 'diff_image', + 'rms', 'tolerance', 'result_image']: summary[k] = summary[k] or baseline_summary.get(k) # Append the log from image comparison @@ -597,9 +616,12 @@ def item_function_wrapper(*args, **kwargs): remove_ticks_and_titles(fig) test_name = self.generate_test_name(item) + result_dir = self.make_test_results_dir(item) summary = { 'status': None, + 'image_status': None, + 'hash_status': None, 'status_msg': None, 'baseline_image': None, 'diff_image': None, @@ -614,21 +636,23 @@ def item_function_wrapper(*args, **kwargs): # reference images or simply running the test. if self.generate_dir is not None: summary['status'] = 'skipped' + summary['image_status'] = 'generated' summary['status_msg'] = 'Skipped test, since generating image.' - self.generate_baseline_image(item, fig) - if self.generate_hash_library is None: - self._test_results[str(pathify(test_name))] = summary - pytest.skip("Skipping test, since generating image.") + generate_image = self.generate_baseline_image(item, fig) + if self.results_always: # Make baseline image available in HTML + result_image = (result_dir / "baseline.png").absolute() + shutil.copy(generate_image, result_image) + summary['baseline_image'] = \ + result_image.relative_to(self.results_dir).as_posix() if self.generate_hash_library is not None: + summary['hash_status'] = 'generated' image_hash = self.generate_image_hash(item, fig) self._generated_hash_library[test_name] = image_hash - summary['result_hash'] = image_hash + summary['baseline_hash'] = image_hash # Only test figures if not generating images if self.generate_dir is None: - result_dir = self.make_test_results_dir(item) - # Compare to hash library if self.hash_library or compare.kwargs.get('hash_library', None): msg = self.compare_image_to_hash_library(item, fig, result_dir, summary=summary) @@ -645,12 +669,15 @@ def item_function_wrapper(*args, **kwargs): for image_type in ['baseline_image', 'diff_image', 'result_image']: summary[image_type] = None # image no longer exists else: - self._test_results[str(pathify(test_name))] = summary + self._test_results[test_name] = summary pytest.fail(msg, pytrace=False) close_mpl_figure(fig) - self._test_results[str(pathify(test_name))] = summary + self._test_results[test_name] = summary + + if summary['status'] == 'skipped': + pytest.skip(summary['status_msg']) if item.cls is not None: setattr(item.cls, item.function.__name__, item_function_wrapper) @@ -667,21 +694,36 @@ def pytest_unconfigure(self, config): """ Save out the hash library at the end of the run. """ + result_hash_library = self.results_dir / (self.results_hash_library_name or "temp.json") if self.generate_hash_library is not None: hash_library_path = Path(config.rootdir) / self.generate_hash_library hash_library_path.parent.mkdir(parents=True, exist_ok=True) with open(hash_library_path, "w") as fp: json.dump(self._generated_hash_library, fp, indent=2) + if self.results_always: # Make accessible in results directory + # Use same name as generated + result_hash_library = self.results_dir / hash_library_path.name + shutil.copy(hash_library_path, result_hash_library) + elif self.results_always and self.results_hash_library_name: + result_hashes = {k: v['result_hash'] for k, v in self._test_results.items() + if v['result_hash']} + if len(result_hashes) > 0: # At least one hash comparison test + with open(result_hash_library, "w") as fp: + json.dump(result_hashes, fp, indent=2) if self.generate_summary: + kwargs = {} if 'json' in self.generate_summary: summary = self.generate_summary_json() print(f"A JSON report can be found at: {summary}") + if result_hash_library.exists(): # link to it in the HTML + kwargs["hash_library"] = result_hash_library.name if 'html' in self.generate_summary: - summary = generate_summary_html(self._test_results, self.results_dir) + summary = generate_summary_html(self._test_results, self.results_dir, **kwargs) print(f"A summary of the failed tests can be found at: {summary}") if 'basic-html' in self.generate_summary: - summary = generate_summary_basic_html(self._test_results, self.results_dir) + summary = generate_summary_basic_html(self._test_results, self.results_dir, + **kwargs) print(f"A summary of the failed tests can be found at: {summary}") diff --git a/pytest_mpl/summary/html.py b/pytest_mpl/summary/html.py index 858cfe21..78fbcc19 100644 --- a/pytest_mpl/summary/html.py +++ b/pytest_mpl/summary/html.py @@ -26,26 +26,12 @@ class Results: def __init__(self, results, title="Image comparison"): self.title = title # HTML Codestin Search App - + {% include 'navbar.html' %}
diff --git a/pytest_mpl/summary/templates/basic.html b/pytest_mpl/summary/templates/basic.html index 3feb1095..27e6417c 100644 --- a/pytest_mpl/summary/templates/basic.html +++ b/pytest_mpl/summary/templates/basic.html @@ -48,6 +48,11 @@

Image test comparison

of those have a matching baseline image {%- endif %}

+{% if hash_library -%} +

+ Download generated hash library +

+{%- endif %} diff --git a/pytest_mpl/summary/templates/extra.js b/pytest_mpl/summary/templates/extra.js index cb80eefe..d5c3c341 100644 --- a/pytest_mpl/summary/templates/extra.js +++ b/pytest_mpl/summary/templates/extra.js @@ -1,10 +1,3 @@ -// Remove all elements of class mpl-hash if hash test not run -if (document.body.classList[0] == 'no-hash-test') { - document.querySelectorAll('.mpl-hash').forEach(function (elem) { - elem.parentElement.removeChild(elem); - }); -} - // Enable tooltips var tooltipTriggerList = [].slice.call(document.querySelectorAll('[data-bs-toggle="tooltip"]')) var tooltipList = tooltipTriggerList.map(function (tooltipTriggerEl) { diff --git a/pytest_mpl/summary/templates/filter.html b/pytest_mpl/summary/templates/filter.html index 3cd8432f..81e282cc 100644 --- a/pytest_mpl/summary/templates/filter.html +++ b/pytest_mpl/summary/templates/filter.html @@ -16,7 +16,7 @@
Sort tests by...
{{ sort_option('status-sort', 'status', 'desc', default=true) }} {{ sort_option('collected-sort', 'collected', 'asc') }} {{ sort_option('test-name', 'name') }} - {% if results.warn_missing['baseline_image'] -%} + {% if results.image_comparison -%} {{ sort_option('rms-sort', 'RMS', 'desc') }} {%- endif %} @@ -34,15 +34,15 @@
Show tests which have...
{{ filter_option('overall-failed', 'failed') }} {{ filter_option('overall-skipped', 'skipped') }} - {% if results.warn_missing['baseline_image'] -%} + {% if results.image_comparison -%}
{{ filter_option('image-match', 'matching images') }} {{ filter_option('image-diff', 'differing images') }} {{ filter_option('image-missing', 'no baseline image') }}
{%- endif %} - {% if results.warn_missing['baseline_hash'] -%} -
+ {% if results.hash_comparison -%} +
{{ filter_option('hash-match', 'matching hashes') }} {{ filter_option('hash-diff', 'differing hashes') }} {{ filter_option('hash-missing', 'no baseline hash') }} diff --git a/pytest_mpl/summary/templates/navbar.html b/pytest_mpl/summary/templates/navbar.html index 3ba762ac..17fa7bfd 100644 --- a/pytest_mpl/summary/templates/navbar.html +++ b/pytest_mpl/summary/templates/navbar.html @@ -9,6 +9,17 @@ + {% if hash_library -%} + + + + + + {%- endif %}
- {%- else -%} + {%- elif r.result_image -%} result image + {%- elif r.baseline_image -%} + baseline image {%- endif %} {% filter indent(width=8) -%} diff --git a/pytest_mpl/summary/templates/result_images.html b/pytest_mpl/summary/templates/result_images.html index 822bbd46..36ab2943 100644 --- a/pytest_mpl/summary/templates/result_images.html +++ b/pytest_mpl/summary/templates/result_images.html @@ -6,14 +6,14 @@
{{ r.module }}
{{ r.name }}
- {% if results.warn_missing['baseline_image'] -%} + {% if r.image_status and r.image_status != 'generated' -%}
{% macro image_card(file, name) -%}
{{ name }}
{% if file -%} - + {{ name }} {%- endif %}
@@ -41,21 +41,25 @@
{{ r.name }}
{%- endfilter %} {%- endmacro -%} - {% if results.warn_missing['baseline_image'] -%} -
+ {% if r.image_status -%} +
{{ r.image_status | image_status_msg }}
+ {% if r.image_status == 'match' or r.image_status == 'diff' -%}
{{ pre_data('rms', r.rms_str, 'RMS') }} {{ pre_data('tolerance', r.tolerance, 'Tolerance') }}
+ {%- endif %}
{%- endif %} - {% if results.warn_missing['baseline_hash'] -%} -
+ {% if r.hash_status -%} +
{{ r.hash_status | hash_status_msg }}
{{ pre_data('baseline_hash', r.baseline_hash, 'Baseline') }} + {% if r.hash_status != 'generated' -%} {{ pre_data('result_hash', r.result_hash, 'Result') }} + {%- endif %}
{%- endif %} diff --git a/pytest_mpl/summary/templates/styles.css b/pytest_mpl/summary/templates/styles.css index 3effb1df..c5870b09 100644 --- a/pytest_mpl/summary/templates/styles.css +++ b/pytest_mpl/summary/templates/styles.css @@ -1,6 +1,3 @@ -body.no-hash-test .mpl-hash { - display: none; -} .navbar-brand span.repo { font-weight: bold; } @@ -14,6 +11,9 @@ body.no-hash-test .mpl-hash { .nav-filtertools .nav-searchbar { flex: 10; } +.nav-filtertools svg { + vertical-align: text-bottom; +} #filterForm .spacer { flex: 1; } @@ -29,12 +29,12 @@ body.no-hash-test .mpl-hash { max-width: 400px; } } -.image-missing .mpl-image .card-body { - display: none; -} pre { white-space: pre-line; } +.tooltip-inner pre { + margin: 0; +} div.result div.status-badge button img { vertical-align: sub; } diff --git a/tests/subtests/helpers.py b/tests/subtests/helpers.py index 0c688843..0f345f03 100644 --- a/tests/subtests/helpers.py +++ b/tests/subtests/helpers.py @@ -9,7 +9,8 @@ class MatchError(Exception): pass -def diff_summary(baseline, result, baseline_hash_library=None, result_hash_library=None): +def diff_summary(baseline, result, baseline_hash_library=None, result_hash_library=None, + generating_hashes=False): """Diff a pytest-mpl summary dictionary. Parameters @@ -26,6 +27,9 @@ def diff_summary(baseline, result, baseline_hash_library=None, result_hash_libra Path to the "baseline" image hash library. Result hashes in the baseline summary are updated to these values to handle different Matplotlib versions. + generating_hashes : bool, optional, default=False + Whether `--mpl-generate-hash-library` was specified and + both of `--mpl-hash-library` and `hash_library=` were not. """ if baseline_hash_library and baseline_hash_library.exists(): # Load "correct" baseline hashes @@ -57,10 +61,13 @@ def diff_summary(baseline, result, baseline_hash_library=None, result_hash_libra # Swap the baseline and result hashes in the summary # for the corresponding hashes in each hash library - if baseline_hash_library and test in baseline_hash_library: + if baseline_hash_library and test in baseline_hash_library and not generating_hashes: baseline_summary = replace_hash(baseline_summary, 'baseline_hash', baseline_hash_library[test]) if result_hash_library: + if generating_hashes: # Newly generate result will appear as baseline_hash + baseline_summary = replace_hash(baseline_summary, 'baseline_hash', + result_hash_library[test]) baseline_summary = replace_hash(baseline_summary, 'result_hash', result_hash_library[test]) diff --git a/tests/subtests/summaries/test_default.json b/tests/subtests/summaries/test_default.json index a4acadc6..b4b51444 100644 --- a/tests/subtests/summaries/test_default.json +++ b/tests/subtests/summaries/test_default.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": null, "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -34,6 +40,8 @@ }, "subtests.subtest.test_hdiff_imatch": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -45,6 +53,8 @@ }, "subtests.subtest.test_hdiff_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", @@ -56,6 +66,8 @@ }, "subtests.subtest.test_hdiff_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": null, "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -67,6 +79,8 @@ }, "subtests.subtest.test_hmissing_imatch": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -78,6 +92,8 @@ }, "subtests.subtest.test_hmissing_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", @@ -89,6 +105,8 @@ }, "subtests.subtest.test_hmissing_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": null, "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -100,6 +118,8 @@ }, "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -111,6 +131,8 @@ }, "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", @@ -122,6 +144,8 @@ }, "subtests.subtest.test_hdiff_imatch_savefig": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -133,6 +157,8 @@ }, "subtests.subtest.test_hdiff_imatch_style": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -144,6 +170,8 @@ }, "subtests.subtest.test_hdiff_imatch_removetext": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, diff --git a/tests/subtests/summaries/test_generate.json b/tests/subtests/summaries/test_generate.json new file mode 100644 index 00000000..6f9f78e2 --- /dev/null +++ b/tests/subtests/summaries/test_generate.json @@ -0,0 +1,184 @@ +{ + "subtests.subtest.test_hmatch_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "4a47c9b7920779cc83eabe2bbb64b9c40745d9d8abfa57857f93a5d8f12a5a03", + "result_hash": null + }, + "subtests.subtest.test_hmatch_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa", + "result_hash": null + }, + "subtests.subtest.test_hmatch_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "e937fa1997d088c904ca35b1ab542e2285ea47b84df976490380f9c5f5b5f8ae", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "2cae8f315d44b06de8f45d937af46a67bd1389edd6e4cde32f9feb4b7472284f", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "afc411cfa34db3a5819ac4127704e86acf27d24d1ea2410718853d3d7e1d6ae0", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "7ee8370efdc4b767634d12355657ca4f2460176670c07b31f3fb72cea0e79856", + "result_hash": null + }, + "subtests.subtest.test_hmissing_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "5101e60ac100cf2c2f418a0a6a382aae0060339e76718730344f539b61f7dc7e", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_savefig": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "8864803a4b4026d8c6dc0ab950228793ea255cd9b6c629c39db9e6315a9af6bc", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_style": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "e3c8de36c2bad7dca131e4cbbfe229f882b5beec62750fb7da29314fd6a1ff13", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_removetext": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "e4c06cf613c6836c1b1202abaae69cf65bc2232a8e31ab1040454bedc8e31e7a", + "result_hash": null + } +} diff --git a/tests/subtests/summaries/test_generate_hashes_only.json b/tests/subtests/summaries/test_generate_hashes_only.json new file mode 100644 index 00000000..8dee25fa --- /dev/null +++ b/tests/subtests/summaries/test_generate_hashes_only.json @@ -0,0 +1,184 @@ +{ + "subtests.subtest.test_hmatch_imatch": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "4a47c9b7920779cc83eabe2bbb64b9c40745d9d8abfa57857f93a5d8f12a5a03", + "result_hash": null + }, + "subtests.subtest.test_hmatch_idiff": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", + "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", + "rms": 11.100353848213828, + "tolerance": 2, + "result_image": "subtests.subtest.test_hmatch_idiff/result.png", + "baseline_hash": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa", + "result_hash": null + }, + "subtests.subtest.test_hmatch_imissing": { + "status": "failed", + "image_status": "missing", + "hash_status": "generated", + "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmatch_imissing/result.png", + "baseline_hash": "e937fa1997d088c904ca35b1ab542e2285ea47b84df976490380f9c5f5b5f8ae", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "2cae8f315d44b06de8f45d937af46a67bd1389edd6e4cde32f9feb4b7472284f", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", + "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", + "rms": 11.182677079602481, + "tolerance": 2, + "result_image": "subtests.subtest.test_hdiff_idiff/result.png", + "baseline_hash": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imissing": { + "status": "failed", + "image_status": "missing", + "hash_status": "generated", + "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hdiff_imissing/result.png", + "baseline_hash": "afc411cfa34db3a5819ac4127704e86acf27d24d1ea2410718853d3d7e1d6ae0", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imatch": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "7ee8370efdc4b767634d12355657ca4f2460176670c07b31f3fb72cea0e79856", + "result_hash": null + }, + "subtests.subtest.test_hmissing_idiff": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", + "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", + "rms": 12.12938597648977, + "tolerance": 2, + "result_image": "subtests.subtest.test_hmissing_idiff/result.png", + "baseline_hash": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imissing": { + "status": "failed", + "image_status": "missing", + "hash_status": "generated", + "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmissing_imissing/result.png", + "baseline_hash": "5101e60ac100cf2c2f418a0a6a382aae0060339e76718730344f539b61f7dc7e", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_tolerance": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 200, + "result_image": null, + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff_tolerance": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", + "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", + "rms": 29.260332173249314, + "tolerance": 3, + "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png", + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_savefig": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "8864803a4b4026d8c6dc0ab950228793ea255cd9b6c629c39db9e6315a9af6bc", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_style": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "e3c8de36c2bad7dca131e4cbbfe229f882b5beec62750fb7da29314fd6a1ff13", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_removetext": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "e4c06cf613c6836c1b1202abaae69cf65bc2232a8e31ab1040454bedc8e31e7a", + "result_hash": null + } +} diff --git a/tests/subtests/summaries/test_generate_images_only.json b/tests/subtests/summaries/test_generate_images_only.json new file mode 100644 index 00000000..7720807d --- /dev/null +++ b/tests/subtests/summaries/test_generate_images_only.json @@ -0,0 +1,184 @@ +{ + "subtests.subtest.test_hmatch_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmatch_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmatch_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmissing_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmissing_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmissing_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_savefig": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_style": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_removetext": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + } +} diff --git a/tests/subtests/summaries/test_hash.json b/tests/subtests/summaries/test_hash.json index bec04c5a..4304bf03 100644 --- a/tests/subtests/summaries/test_hash.json +++ b/tests/subtests/summaries/test_hash.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -35,6 +41,8 @@ "subtests.subtest.test_hdiff_imatch": { "status": "failed", "status_msg": "REGEX:Hash 6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 doesn't match hash d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -46,6 +54,8 @@ "subtests.subtest.test_hdiff_idiff": { "status": "failed", "status_msg": "REGEX:Hash 443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 doesn't match hash d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -57,6 +67,8 @@ "subtests.subtest.test_hdiff_imissing": { "status": "failed", "status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -68,6 +80,8 @@ "subtests.subtest.test_hmissing_imatch": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824\\.", + "image_status": null, + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -79,6 +93,8 @@ "subtests.subtest.test_hmissing_idiff": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab\\.", + "image_status": null, + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -90,6 +106,8 @@ "subtests.subtest.test_hmissing_imissing": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.", + "image_status": null, + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -101,6 +119,8 @@ "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -112,6 +132,8 @@ "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -123,6 +145,8 @@ "subtests.subtest.test_hdiff_imatch_savefig": { "status": "failed", "status_msg": "REGEX:Hash 5dc1c2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 doesn't match hash d1ffc2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -134,6 +158,8 @@ "subtests.subtest.test_hdiff_imatch_style": { "status": "failed", "status_msg": "REGEX:Hash 185ed1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 doesn't match hash d1ffd1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -145,6 +171,8 @@ "subtests.subtest.test_hdiff_imatch_removetext": { "status": "failed", "status_msg": "REGEX:Hash be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e doesn't match hash d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, diff --git a/tests/subtests/summaries/test_html_generate.json b/tests/subtests/summaries/test_html_generate.json new file mode 100644 index 00000000..84a8e8b9 --- /dev/null +++ b/tests/subtests/summaries/test_html_generate.json @@ -0,0 +1,184 @@ +{ + "subtests.subtest.test_hmatch_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "4a47c9b7920779cc83eabe2bbb64b9c40745d9d8abfa57857f93a5d8f12a5a03", + "result_hash": null + }, + "subtests.subtest.test_hmatch_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa", + "result_hash": null + }, + "subtests.subtest.test_hmatch_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmatch_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "e937fa1997d088c904ca35b1ab542e2285ea47b84df976490380f9c5f5b5f8ae", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "2cae8f315d44b06de8f45d937af46a67bd1389edd6e4cde32f9feb4b7472284f", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "afc411cfa34db3a5819ac4127704e86acf27d24d1ea2410718853d3d7e1d6ae0", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "7ee8370efdc4b767634d12355657ca4f2460176670c07b31f3fb72cea0e79856", + "result_hash": null + }, + "subtests.subtest.test_hmissing_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmissing_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "5101e60ac100cf2c2f418a0a6a382aae0060339e76718730344f539b61f7dc7e", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_savefig": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "8864803a4b4026d8c6dc0ab950228793ea255cd9b6c629c39db9e6315a9af6bc", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_style": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "e3c8de36c2bad7dca131e4cbbfe229f882b5beec62750fb7da29314fd6a1ff13", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_removetext": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "e4c06cf613c6836c1b1202abaae69cf65bc2232a8e31ab1040454bedc8e31e7a", + "result_hash": null + } +} diff --git a/tests/subtests/summaries/test_html_generate_hashes_only.json b/tests/subtests/summaries/test_html_generate_hashes_only.json new file mode 100644 index 00000000..0a813734 --- /dev/null +++ b/tests/subtests/summaries/test_html_generate_hashes_only.json @@ -0,0 +1,184 @@ +{ + "subtests.subtest.test_hmatch_imatch": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_hmatch_imatch/result.png", + "baseline_hash": "4a47c9b7920779cc83eabe2bbb64b9c40745d9d8abfa57857f93a5d8f12a5a03", + "result_hash": null + }, + "subtests.subtest.test_hmatch_idiff": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", + "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", + "rms": 11.100353848213828, + "tolerance": 2, + "result_image": "subtests.subtest.test_hmatch_idiff/result.png", + "baseline_hash": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa", + "result_hash": null + }, + "subtests.subtest.test_hmatch_imissing": { + "status": "failed", + "image_status": "missing", + "hash_status": "generated", + "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmatch_imissing/result.png", + "baseline_hash": "e937fa1997d088c904ca35b1ab542e2285ea47b84df976490380f9c5f5b5f8ae", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_hdiff_imatch/result.png", + "baseline_hash": "2cae8f315d44b06de8f45d937af46a67bd1389edd6e4cde32f9feb4b7472284f", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", + "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", + "rms": 11.182677079602481, + "tolerance": 2, + "result_image": "subtests.subtest.test_hdiff_idiff/result.png", + "baseline_hash": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imissing": { + "status": "failed", + "image_status": "missing", + "hash_status": "generated", + "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hdiff_imissing/result.png", + "baseline_hash": "afc411cfa34db3a5819ac4127704e86acf27d24d1ea2410718853d3d7e1d6ae0", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imatch": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_hmissing_imatch/result.png", + "baseline_hash": "7ee8370efdc4b767634d12355657ca4f2460176670c07b31f3fb72cea0e79856", + "result_hash": null + }, + "subtests.subtest.test_hmissing_idiff": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", + "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", + "rms": 12.12938597648977, + "tolerance": 2, + "result_image": "subtests.subtest.test_hmissing_idiff/result.png", + "baseline_hash": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imissing": { + "status": "failed", + "image_status": "missing", + "hash_status": "generated", + "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmissing_imissing/result.png", + "baseline_hash": "5101e60ac100cf2c2f418a0a6a382aae0060339e76718730344f539b61f7dc7e", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_tolerance": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 200, + "result_image": "subtests.subtest.test_hdiff_imatch_tolerance/result.png", + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff_tolerance": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", + "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", + "rms": 29.260332173249314, + "tolerance": 3, + "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png", + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_savefig": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_hdiff_imatch_savefig/result.png", + "baseline_hash": "8864803a4b4026d8c6dc0ab950228793ea255cd9b6c629c39db9e6315a9af6bc", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_style": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_hdiff_imatch_style/result.png", + "baseline_hash": "e3c8de36c2bad7dca131e4cbbfe229f882b5beec62750fb7da29314fd6a1ff13", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_removetext": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_hdiff_imatch_removetext/result.png", + "baseline_hash": "e4c06cf613c6836c1b1202abaae69cf65bc2232a8e31ab1040454bedc8e31e7a", + "result_hash": null + } +} diff --git a/tests/subtests/summaries/test_html_generate_images_only.json b/tests/subtests/summaries/test_html_generate_images_only.json new file mode 100644 index 00000000..bed75fae --- /dev/null +++ b/tests/subtests/summaries/test_html_generate_images_only.json @@ -0,0 +1,184 @@ +{ + "subtests.subtest.test_hmatch_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmatch_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmatch_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmatch_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmissing_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmissing_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmissing_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmissing_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_savefig": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_style": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_removetext": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + } +} diff --git a/tests/subtests/summaries/test_html_hashes_only.json b/tests/subtests/summaries/test_html_hashes_only.json index 448ce9c2..a55666ac 100644 --- a/tests/subtests/summaries/test_html_hashes_only.json +++ b/tests/subtests/summaries/test_html_hashes_only.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -35,6 +41,8 @@ "subtests.subtest.test_hdiff_imatch": { "status": "failed", "status_msg": "REGEX:Hash 6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 doesn't match hash d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -46,6 +54,8 @@ "subtests.subtest.test_hdiff_idiff": { "status": "failed", "status_msg": "REGEX:Hash 443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 doesn't match hash d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -57,6 +67,8 @@ "subtests.subtest.test_hdiff_imissing": { "status": "failed", "status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -68,6 +80,8 @@ "subtests.subtest.test_hmissing_imatch": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824\\.", + "image_status": null, + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -79,6 +93,8 @@ "subtests.subtest.test_hmissing_idiff": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab\\.", + "image_status": null, + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -90,6 +106,8 @@ "subtests.subtest.test_hmissing_imissing": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.", + "image_status": null, + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -101,6 +119,8 @@ "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -112,6 +132,8 @@ "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -123,6 +145,8 @@ "subtests.subtest.test_hdiff_imatch_savefig": { "status": "failed", "status_msg": "REGEX:Hash 5dc1c2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 doesn't match hash d1ffc2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -134,6 +158,8 @@ "subtests.subtest.test_hdiff_imatch_style": { "status": "failed", "status_msg": "REGEX:Hash 185ed1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 doesn't match hash d1ffd1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -145,6 +171,8 @@ "subtests.subtest.test_hdiff_imatch_removetext": { "status": "failed", "status_msg": "REGEX:Hash be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e doesn't match hash d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, diff --git a/tests/subtests/summaries/test_html_images_only.json b/tests/subtests/summaries/test_html_images_only.json index c8a5058c..0b33035e 100644 --- a/tests/subtests/summaries/test_html_images_only.json +++ b/tests/subtests/summaries/test_html_images_only.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": null, "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -34,6 +40,8 @@ }, "subtests.subtest.test_hdiff_imatch": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", "diff_image": null, @@ -45,6 +53,8 @@ }, "subtests.subtest.test_hdiff_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", @@ -56,6 +66,8 @@ }, "subtests.subtest.test_hdiff_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": null, "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -67,6 +79,8 @@ }, "subtests.subtest.test_hmissing_imatch": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", "diff_image": null, @@ -78,6 +92,8 @@ }, "subtests.subtest.test_hmissing_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", @@ -89,6 +105,8 @@ }, "subtests.subtest.test_hmissing_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": null, "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -100,6 +118,8 @@ }, "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", "diff_image": null, @@ -111,6 +131,8 @@ }, "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", @@ -122,6 +144,8 @@ }, "subtests.subtest.test_hdiff_imatch_savefig": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", "diff_image": null, @@ -133,6 +157,8 @@ }, "subtests.subtest.test_hdiff_imatch_style": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", "diff_image": null, @@ -144,6 +170,8 @@ }, "subtests.subtest.test_hdiff_imatch_removetext": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", "diff_image": null, diff --git a/tests/subtests/summaries/test_hybrid.json b/tests/subtests/summaries/test_hybrid.json index 2f2930b0..851e67f1 100644 --- a/tests/subtests/summaries/test_hybrid.json +++ b/tests/subtests/summaries/test_hybrid.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -35,6 +41,8 @@ "subtests.subtest.test_hdiff_imatch": { "status": "failed", "status_msg": "REGEX:Hash 6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 doesn't match hash d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", "diff_image": null, "rms": null, @@ -46,6 +54,8 @@ "subtests.subtest.test_hdiff_idiff": { "status": "failed", "status_msg": "REGEX:Hash 443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 doesn't match hash d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "image_status": "diff", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", "rms": 11.182677079602481, @@ -57,6 +67,8 @@ "subtests.subtest.test_hdiff_imissing": { "status": "failed", "status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "image_status": "missing", + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -68,6 +80,8 @@ "subtests.subtest.test_hmissing_imatch": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "missing", "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", "diff_image": null, "rms": null, @@ -79,6 +93,8 @@ "subtests.subtest.test_hmissing_idiff": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "image_status": "diff", + "hash_status": "missing", "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", "rms": 12.12938597648977, @@ -90,6 +106,8 @@ "subtests.subtest.test_hmissing_imissing": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "image_status": "missing", + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -101,6 +119,8 @@ "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", "diff_image": null, "rms": null, @@ -112,6 +132,8 @@ "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", + "image_status": "diff", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", "rms": 29.260332173249314, @@ -123,6 +145,8 @@ "subtests.subtest.test_hdiff_imatch_savefig": { "status": "failed", "status_msg": "REGEX:Hash 5dc1c2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 doesn't match hash d1ffc2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", "diff_image": null, "rms": null, @@ -134,6 +158,8 @@ "subtests.subtest.test_hdiff_imatch_style": { "status": "failed", "status_msg": "REGEX:Hash 185ed1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 doesn't match hash d1ffd1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", "diff_image": null, "rms": null, @@ -145,6 +171,8 @@ "subtests.subtest.test_hdiff_imatch_removetext": { "status": "failed", "status_msg": "REGEX:Hash be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e doesn't match hash d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", "diff_image": null, "rms": null, diff --git a/tests/subtests/summaries/test_results_always.json b/tests/subtests/summaries/test_results_always.json index c42fee29..ea7af1d2 100644 --- a/tests/subtests/summaries/test_results_always.json +++ b/tests/subtests/summaries/test_results_always.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "passed", + "image_status": "match", + "hash_status": "match", "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "passed", + "image_status": "diff", + "hash_status": "match", "status_msg": "REGEX:Test hash matches baseline hash\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "passed", + "image_status": "missing", + "hash_status": "match", "status_msg": "REGEX:Test hash matches baseline hash\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -35,6 +41,8 @@ "subtests.subtest.test_hdiff_imatch": { "status": "failed", "status_msg": "REGEX:Hash 6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 doesn't match hash d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", "diff_image": null, "rms": null, @@ -46,6 +54,8 @@ "subtests.subtest.test_hdiff_idiff": { "status": "failed", "status_msg": "REGEX:Hash 443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 doesn't match hash d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "image_status": "diff", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", "rms": 11.182677079602481, @@ -57,6 +67,8 @@ "subtests.subtest.test_hdiff_imissing": { "status": "failed", "status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "image_status": "missing", + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -68,6 +80,8 @@ "subtests.subtest.test_hmissing_imatch": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "missing", "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", "diff_image": null, "rms": null, @@ -79,6 +93,8 @@ "subtests.subtest.test_hmissing_idiff": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "image_status": "diff", + "hash_status": "missing", "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", "rms": 12.12938597648977, @@ -90,6 +106,8 @@ "subtests.subtest.test_hmissing_imissing": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "image_status": "missing", + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -101,6 +119,8 @@ "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", "diff_image": null, "rms": null, @@ -112,6 +132,8 @@ "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", + "image_status": "diff", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", "rms": 29.260332173249314, @@ -123,6 +145,8 @@ "subtests.subtest.test_hdiff_imatch_savefig": { "status": "failed", "status_msg": "REGEX:Hash 5dc1c2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 doesn't match hash d1ffc2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", "diff_image": null, "rms": null, @@ -134,6 +158,8 @@ "subtests.subtest.test_hdiff_imatch_style": { "status": "failed", "status_msg": "REGEX:Hash 185ed1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 doesn't match hash d1ffd1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", "diff_image": null, "rms": null, @@ -145,6 +171,8 @@ "subtests.subtest.test_hdiff_imatch_removetext": { "status": "failed", "status_msg": "REGEX:Hash be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e doesn't match hash d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", "diff_image": null, "rms": null, diff --git a/tests/subtests/test_subtest.py b/tests/subtests/test_subtest.py index 48a2f461..eb85aa5c 100644 --- a/tests/subtests/test_subtest.py +++ b/tests/subtests/test_subtest.py @@ -34,6 +34,7 @@ def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=True, + has_result_hashes=False, generating_hashes=False, update_baseline=UPDATE_BASELINE, update_summary=UPDATE_SUMMARY): """ Run pytest (within pytest) and check JSON summary report. @@ -49,6 +50,12 @@ def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=Tru Summaries to generate in addition to `json`. xfail : bool, optional, default=True Whether the overall pytest run should fail. + has_result_hashes : bool or str, optional, default=False + Whether a hash library is expected to exist in the results directory. + If a string, this is the name of the expected results file. + generating_hashes : bool, optional, default=False + Whether `--mpl-generate-hash-library` was specified and + both of `--mpl-hash-library` and `hash_library=` were not. """ # Parse arguments if summaries is None: @@ -105,11 +112,30 @@ def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=Tru # Compare summaries diff_summary(baseline_summary, result_summary, - baseline_hash_library=HASH_LIBRARY, result_hash_library=RESULT_LIBRARY) + baseline_hash_library=HASH_LIBRARY, result_hash_library=RESULT_LIBRARY, + generating_hashes=generating_hashes) # Ensure reported images exist assert_existence(result_summary, path=results_path) + # Get expected name for the hash library saved to the results directory + if isinstance(has_result_hashes, str): + result_hash_file = tmp_path / 'results' / has_result_hashes + has_result_hashes = True # convert to bool after processing str + else: + result_hash_file = tmp_path / 'results' / HASH_LIBRARY.name + + # Compare the generated hash library to the expected hash library + if has_result_hashes: + assert result_hash_file.exists() + with open(RESULT_LIBRARY, "r") as f: + baseline = json.load(f) + with open(result_hash_file, "r") as f: + result = json.load(f) + diff_summary({'a': baseline}, {'a': result}) + else: + assert not result_hash_file.exists() + def test_default(tmp_path): run_subtest('test_default', tmp_path, []) @@ -128,13 +154,15 @@ def test_hybrid(tmp_path): @pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") def test_results_always(tmp_path): run_subtest('test_results_always', tmp_path, - [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS, '--mpl-results-always']) + [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS, '--mpl-results-always'], + has_result_hashes=True) @pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") def test_html(tmp_path): run_subtest('test_results_always', tmp_path, - [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS], summaries=['html']) + [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS], summaries=['html'], + has_result_hashes=True) assert (tmp_path / 'results' / 'fig_comparison.html').exists() assert (tmp_path / 'results' / 'extra.js').exists() assert (tmp_path / 'results' / 'styles.css').exists() @@ -142,7 +170,8 @@ def test_html(tmp_path): @pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") def test_html_hashes_only(tmp_path): - run_subtest('test_html_hashes_only', tmp_path, [HASH_LIBRARY_FLAG], summaries=['html']) + run_subtest('test_html_hashes_only', tmp_path, [HASH_LIBRARY_FLAG], summaries=['html'], + has_result_hashes=True) assert (tmp_path / 'results' / 'fig_comparison.html').exists() assert (tmp_path / 'results' / 'extra.js').exists() assert (tmp_path / 'results' / 'styles.css').exists() @@ -158,5 +187,67 @@ def test_html_images_only(tmp_path): @pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") def test_basic_html(tmp_path): run_subtest('test_results_always', tmp_path, - [HASH_LIBRARY_FLAG, *BASELINE_IMAGES_FLAG_REL], summaries=['basic-html']) + [HASH_LIBRARY_FLAG, *BASELINE_IMAGES_FLAG_REL], summaries=['basic-html'], + has_result_hashes=True) assert (tmp_path / 'results' / 'fig_comparison_basic.html').exists() + + +@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") +def test_generate(tmp_path): + # generating hashes and images; no testing + run_subtest('test_generate', tmp_path, + [rf'--mpl-generate-path={tmp_path}', + rf'--mpl-generate-hash-library={tmp_path / "test_hashes.json"}'], + xfail=False, generating_hashes=True) + + +def test_generate_images_only(tmp_path): + # generating images; no testing + run_subtest('test_generate_images_only', tmp_path, + [rf'--mpl-generate-path={tmp_path}'], xfail=False) + + +@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") +def test_generate_hashes_only(tmp_path): + # generating hashes; testing images + run_subtest('test_generate_hashes_only', tmp_path, + [rf'--mpl-generate-hash-library={tmp_path / "test_hashes.json"}'], + generating_hashes=True) + + +@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") +def test_html_generate(tmp_path): + # generating hashes and images; no testing + run_subtest('test_html_generate', tmp_path, + [rf'--mpl-generate-path={tmp_path}', + rf'--mpl-generate-hash-library={tmp_path / "test_hashes.json"}'], + summaries=['html'], xfail=False, has_result_hashes="test_hashes.json", + generating_hashes=True) + assert (tmp_path / 'results' / 'fig_comparison.html').exists() + + +def test_html_generate_images_only(tmp_path): + # generating images; no testing + run_subtest('test_html_generate_images_only', tmp_path, + [rf'--mpl-generate-path={tmp_path}'], + summaries=['html'], xfail=False) + assert (tmp_path / 'results' / 'fig_comparison.html').exists() + + +@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") +def test_html_generate_hashes_only(tmp_path): + # generating hashes; testing images + run_subtest('test_html_generate_hashes_only', tmp_path, + [rf'--mpl-generate-hash-library={tmp_path / "test_hashes.json"}'], + summaries=['html'], has_result_hashes="test_hashes.json", generating_hashes=True) + assert (tmp_path / 'results' / 'fig_comparison.html').exists() + + +@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") +def test_html_run_generate_hashes_only(tmp_path): + # generating hashes; testing hashes + run_subtest('test_html_hashes_only', tmp_path, + [rf'--mpl-generate-hash-library={tmp_path / "test_hashes.json"}', + HASH_LIBRARY_FLAG], + summaries=['html'], has_result_hashes="test_hashes.json") + assert (tmp_path / 'results' / 'fig_comparison.html').exists()