From 536900b7e3464f65fec4a9bc1e1d6d26d6f9866f Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Fri, 7 Jan 2022 17:53:08 +0000 Subject: [PATCH 01/24] Add bootstrap template Signed-off-by: Conor MacBride --- pytest_mpl/summary/__init__.py | 0 pytest_mpl/summary/html.py | 204 ++++++++++++++++++ pytest_mpl/summary/templates/base.html | 27 +++ pytest_mpl/summary/templates/extra.js | 12 ++ pytest_mpl/summary/templates/filter.html | 66 ++++++ pytest_mpl/summary/templates/hash.svg | 3 + pytest_mpl/summary/templates/image.svg | 4 + pytest_mpl/summary/templates/navbar.html | 47 ++++ pytest_mpl/summary/templates/result.html | 13 ++ .../summary/templates/result_badge.html | 7 + .../summary/templates/result_badge_icon.html | 6 + .../summary/templates/result_diffimage.html | 8 + .../summary/templates/result_images.html | 68 ++++++ pytest_mpl/summary/templates/styles.css | 30 +++ 14 files changed, 495 insertions(+) create mode 100644 pytest_mpl/summary/__init__.py create mode 100644 pytest_mpl/summary/html.py create mode 100644 pytest_mpl/summary/templates/base.html create mode 100644 pytest_mpl/summary/templates/extra.js create mode 100644 pytest_mpl/summary/templates/filter.html create mode 100644 pytest_mpl/summary/templates/hash.svg create mode 100644 pytest_mpl/summary/templates/image.svg create mode 100644 pytest_mpl/summary/templates/navbar.html create mode 100644 pytest_mpl/summary/templates/result.html create mode 100644 pytest_mpl/summary/templates/result_badge.html create mode 100644 pytest_mpl/summary/templates/result_badge_icon.html create mode 100644 pytest_mpl/summary/templates/result_diffimage.html create mode 100644 pytest_mpl/summary/templates/result_images.html create mode 100644 pytest_mpl/summary/templates/styles.css diff --git a/pytest_mpl/summary/__init__.py b/pytest_mpl/summary/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pytest_mpl/summary/html.py b/pytest_mpl/summary/html.py new file mode 100644 index 00000000..54d49566 --- /dev/null +++ b/pytest_mpl/summary/html.py @@ -0,0 +1,204 @@ +import os +import shutil + +__all__ = ['generate_summary_html'] + +BTN_CLASS = { + 'passed': 'success', + 'failed': 'danger', + 'skipped': 'warning', + 'match': 'success', + 'diff': 'danger', + 'missing': 'warning', +} + +IMAGE_STATUS = { + 'match': 'Baseline image matches', + 'diff': 'Baseline image differs', + 'missing': 'Baseline image not found', +} + +HASH_STATUS = { + 'match': 'Baseline hash matches', + 'diff': 'Baseline hash differs', + 'missing': 'Baseline hash not found', +} + + +def template(name): + file = os.path.join(os.path.dirname(__file__), 'templates', f'{name}.html') + f = open(file, 'r') + return f.read() + + +BASE = template('base') +NAVBAR = template('navbar') +FILTER = template('filter') +RESULT = template('result') +RESULT_DIFFIMAGE = template('result_diffimage') +RESULT_BADGE = template('result_badge') +RESULT_BADGE_ICON = template('result_badge_icon') +RESULT_IMAGES = template('result_images') + + +def get_status(item, card_id, warn_missing): + status = { + 'overall': None, + 'image': None, + 'hash': None, + } + + assert item['status'] in BTN_CLASS.keys() + status['overall'] = item['status'] + + if item['rms'] is None and item['tolerance'] is not None: + status['image'] = 'match' + elif item['rms'] is not None: + status['image'] = 'diff' + elif item['baseline_image'] is None: + status['image'] = 'missing' + else: + raise ValueError('Unknown image result.') + + baseline_hash = item['baseline_hash'] + result_hash = item['result_hash'] + if baseline_hash is not None or result_hash is not None: + if baseline_hash is None: + status['hash'] = 'missing' + elif baseline_hash == result_hash: + status['hash'] = 'match' + else: + status['hash'] = 'diff' + + classes = [f'{k}-{str(v).lower()}' for k, v in status.items()] + + extra_badges = '' + for test_type, status_dict in [('image', IMAGE_STATUS), ('hash', HASH_STATUS)]: + if not warn_missing[f'baseline_{test_type}']: + continue # not expected to exist + if ( + (status[test_type] == 'missing') or + (status['overall'] == 'failed' and status[test_type] == 'match') or + (status['overall'] == 'passed' and status[test_type] == 'diff') + ): + extra_badges += RESULT_BADGE_ICON.format( + card_id=card_id, + btn_class=BTN_CLASS[status[test_type]], + svg=test_type, + tooltip=status_dict[status[test_type]], + ) + + badge = RESULT_BADGE.format( + card_id=card_id, + status=status['overall'].upper(), + btn_class=BTN_CLASS[status['overall']], + extra_badges=extra_badges, + ) + + return status, classes, badge + + +def card(name, item, warn_missing=None): + card_id = name.replace('.', '-') + test_name = name.split('.')[-1] + module = '.'.join(name.split('.')[:-1]) + + status, classes, badge = get_status(item, card_id, warn_missing) + + if item['diff_image'] is None: + image = f'result image' + else: # show overlapping diff and result images + image = RESULT_DIFFIMAGE.format(diff=item['diff_image'], result=item["result_image"]) + + image_html = {} + for image_type in ['baseline_image', 'diff_image', 'result_image']: + if item[image_type] is not None: + image_html[image_type] = f'' + else: + image_html[image_type] = '' + + if status['image'] == 'match': + rms = '< tolerance' + else: + rms = item['rms'] + + offcanvas = RESULT_IMAGES.format( + + id=card_id, + test_name=test_name, + module=module, + + baseline_image=image_html['baseline_image'], + diff_image=image_html['diff_image'], + result_image=image_html['result_image'], + + status=status['overall'].upper(), + btn_class=BTN_CLASS[status['overall']], + status_msg=item['status_msg'], + + image_status=IMAGE_STATUS[status['image']], + image_btn_class=BTN_CLASS[status['image']], + rms=rms, + tolerance=item['tolerance'], + + hash_status=HASH_STATUS[status['hash']], + hash_btn_class=BTN_CLASS[status['hash']], + baseline_hash=item['baseline_hash'], + result_hash=item['result_hash'], + + ) + + result_card = RESULT.format( + + classes=" ".join(classes), + + id=card_id, + test_name=test_name, + module=module, + + image=image, + badge=badge, + offcanvas=offcanvas, + + ) + + return result_card + + +def generate_summary_html(results, results_dir): + # If any baseline images or baseline hashes are present, + # assume all results should have them + warn_missing = {'baseline_image': False, 'baseline_hash': False} + for key in warn_missing.keys(): + for result in results.values(): + if result[key] is not None: + warn_missing[key] = True + break + + classes = [] + if warn_missing['baseline_hash'] is False: + classes += ['no-hash-test'] + + # Generate result cards + cards = '' + for name, item in results.items(): + cards += card(name, item, warn_missing=warn_missing) + + # Generate HTML + html = BASE.format( + title="Image comparison", + navbar=NAVBAR, + cards=cards, + filter=FILTER, + classes=" ".join(classes), + ) + + # Write files + for file in ['styles.css', 'extra.js', 'hash.svg', 'image.svg']: + path = os.path.join(os.path.dirname(__file__), 'templates', file) + shutil.copy(path, results_dir / file) + html_file = results_dir / 'fig_comparison.html' + with open(html_file, 'w') as f: + f.write(html) + + return html_file diff --git a/pytest_mpl/summary/templates/base.html b/pytest_mpl/summary/templates/base.html new file mode 100644 index 00000000..7d6681ca --- /dev/null +++ b/pytest_mpl/summary/templates/base.html @@ -0,0 +1,27 @@ + + + + + + + + Codestin Search App + + +{navbar} +
+
+
+ {cards} +
+
+
+
+{filter} + + + + diff --git a/pytest_mpl/summary/templates/extra.js b/pytest_mpl/summary/templates/extra.js new file mode 100644 index 00000000..7c3e0932 --- /dev/null +++ b/pytest_mpl/summary/templates/extra.js @@ -0,0 +1,12 @@ +// Remove all elements of class mpl-hash if hash test not run +if (document.body.classList[0] == 'no-hash-test') { + document.querySelectorAll('.mpl-hash').forEach(function (elem) { + elem.parentElement.removeChild(elem); + }); +} + +// Enable tooltips +var tooltipTriggerList = [].slice.call(document.querySelectorAll('[data-bs-toggle="tooltip"]')) +var tooltipList = tooltipTriggerList.map(function (tooltipTriggerEl) { + return new bootstrap.Tooltip(tooltipTriggerEl) +}) diff --git a/pytest_mpl/summary/templates/filter.html b/pytest_mpl/summary/templates/filter.html new file mode 100644 index 00000000..12e630a8 --- /dev/null +++ b/pytest_mpl/summary/templates/filter.html @@ -0,0 +1,66 @@ +
+
+
Filter results
+ +
+
+
Sort tests by...
+
+ + + + +
+
Show tests which have...
+
+ + + +
+
+ + +
+
+ + +
+
+ + +
+
+ + + + +
+
+
diff --git a/pytest_mpl/summary/templates/hash.svg b/pytest_mpl/summary/templates/hash.svg new file mode 100644 index 00000000..2843ea7c --- /dev/null +++ b/pytest_mpl/summary/templates/hash.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/pytest_mpl/summary/templates/image.svg b/pytest_mpl/summary/templates/image.svg new file mode 100644 index 00000000..e5cc5526 --- /dev/null +++ b/pytest_mpl/summary/templates/image.svg @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/pytest_mpl/summary/templates/navbar.html b/pytest_mpl/summary/templates/navbar.html new file mode 100644 index 00000000..7211e59a --- /dev/null +++ b/pytest_mpl/summary/templates/navbar.html @@ -0,0 +1,47 @@ + diff --git a/pytest_mpl/summary/templates/result.html b/pytest_mpl/summary/templates/result.html new file mode 100644 index 00000000..c4c5702f --- /dev/null +++ b/pytest_mpl/summary/templates/result.html @@ -0,0 +1,13 @@ +
+
+ {image} + {offcanvas} +
+
{module}
+
{test_name}
+
+ {badge} +
+
+
+
diff --git a/pytest_mpl/summary/templates/result_badge.html b/pytest_mpl/summary/templates/result_badge.html new file mode 100644 index 00000000..8e689149 --- /dev/null +++ b/pytest_mpl/summary/templates/result_badge.html @@ -0,0 +1,7 @@ +
+ + {extra_badges} +
diff --git a/pytest_mpl/summary/templates/result_badge_icon.html b/pytest_mpl/summary/templates/result_badge_icon.html new file mode 100644 index 00000000..d90c94ea --- /dev/null +++ b/pytest_mpl/summary/templates/result_badge_icon.html @@ -0,0 +1,6 @@ + diff --git a/pytest_mpl/summary/templates/result_diffimage.html b/pytest_mpl/summary/templates/result_diffimage.html new file mode 100644 index 00000000..8ec9c1b1 --- /dev/null +++ b/pytest_mpl/summary/templates/result_diffimage.html @@ -0,0 +1,8 @@ +
+
+ diff image +
+
+ result image +
+
diff --git a/pytest_mpl/summary/templates/result_images.html b/pytest_mpl/summary/templates/result_images.html new file mode 100644 index 00000000..f19c1603 --- /dev/null +++ b/pytest_mpl/summary/templates/result_images.html @@ -0,0 +1,68 @@ +
+
+
{module}
+ +
+
+
{test_name}
+
+
+
+
Baseline
+ {baseline_image} +
+
+
+
+
Diff
+ {diff_image} +
+
+
+
+
Result
+ {result_image} +
+
+
+
+
+
+
{status}
+
+ 
{status_msg}
+
+
+
+
+
+
{image_status}
+
+
+ + 
{rms}
+
+
+ + 
{tolerance}
+
+
+
+
+
{hash_status}
+
+
+ + 
{baseline_hash}
+
+
+ + 
{result_hash}
+
+
+
+
+
+
+
diff --git a/pytest_mpl/summary/templates/styles.css b/pytest_mpl/summary/templates/styles.css new file mode 100644 index 00000000..970b8622 --- /dev/null +++ b/pytest_mpl/summary/templates/styles.css @@ -0,0 +1,30 @@ +body.no-hash-test .mpl-hash { + display: none; +} +.image-missing .mpl-image .card-body { + display: none; +} +pre { + white-space: pre-wrap; +} +div.result div.status-badge button img { + vertical-align: sub; +} +div.result div.status-badge button.btn-warning img { + filter: brightness(0); +} +div.mpl-images { + height: 1000px; +} +div.hover-image { + position: relative; + border: 3px solid #dc3545; + border-radius: .2rem; +} +div.hover-image div.result-image { + position: absolute; + top: 0; +} +div.hover-image div.result-image img { + filter: opacity(0.3); +} From 06fce93dff77399156acc87d194c92ad86a6ccd9 Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Fri, 7 Jan 2022 19:31:18 +0000 Subject: [PATCH 02/24] Add basic search and sort Signed-off-by: Conor MacBride --- pytest_mpl/summary/html.py | 16 ++++++++++++++++ pytest_mpl/summary/templates/base.html | 7 +++++-- pytest_mpl/summary/templates/extra.js | 7 +++++++ pytest_mpl/summary/templates/filter.html | 6 ++++-- pytest_mpl/summary/templates/navbar.html | 3 +-- pytest_mpl/summary/templates/result.html | 2 ++ pytest_mpl/summary/templates/result_images.html | 6 +++--- 7 files changed, 38 insertions(+), 9 deletions(-) diff --git a/pytest_mpl/summary/html.py b/pytest_mpl/summary/html.py index 54d49566..2aad35ef 100644 --- a/pytest_mpl/summary/html.py +++ b/pytest_mpl/summary/html.py @@ -41,6 +41,21 @@ def template(name): RESULT_IMAGES = template('result_images') +def status_sort(status): + s = 50 + if status['overall'] == 'failed': + s -= 10 + if status['image'] == 'diff': + s -= 3 + elif status['image'] == 'missing': + s -= 4 + if status['hash'] == 'diff': + s -= 1 + elif status['hash'] == 'missing': + s -= 5 + return s + + def get_status(item, card_id, warn_missing): status = { 'overall': None, @@ -155,6 +170,7 @@ def card(name, item, warn_missing=None): id=card_id, test_name=test_name, module=module, + status_sort=status_sort(status), image=image, badge=badge, diff --git a/pytest_mpl/summary/templates/base.html b/pytest_mpl/summary/templates/base.html index 7d6681ca..2d6a5af5 100644 --- a/pytest_mpl/summary/templates/base.html +++ b/pytest_mpl/summary/templates/base.html @@ -8,11 +8,11 @@ integrity="sha384-1BmE4kWBq78iYhFldvKuhfTAU6auU8tT94WrHftjDbrCEXSU1oBoqyl2QvZ6jIW3" crossorigin="anonymous"> Codestin Search App - + {navbar}
-
+
{cards}
@@ -22,6 +22,9 @@ + diff --git a/pytest_mpl/summary/templates/extra.js b/pytest_mpl/summary/templates/extra.js index 7c3e0932..de5c9d15 100644 --- a/pytest_mpl/summary/templates/extra.js +++ b/pytest_mpl/summary/templates/extra.js @@ -10,3 +10,10 @@ var tooltipTriggerList = [].slice.call(document.querySelectorAll('[data-bs-toggl var tooltipList = tooltipTriggerList.map(function (tooltipTriggerEl) { return new bootstrap.Tooltip(tooltipTriggerEl) }) + +// Search, sort and filter +var options = { + valueNames: [ 'test-name', 'status-sort', 'rms', 'baseline-hash', 'result-hash' ] +}; +var userList = new List('results', options); +userList.sort('status-sort'); diff --git a/pytest_mpl/summary/templates/filter.html b/pytest_mpl/summary/templates/filter.html index 12e630a8..e4a46e61 100644 --- a/pytest_mpl/summary/templates/filter.html +++ b/pytest_mpl/summary/templates/filter.html @@ -6,10 +6,12 @@
Filter results
Sort tests by...
- + - + + +
Show tests which have...
diff --git a/pytest_mpl/summary/templates/navbar.html b/pytest_mpl/summary/templates/navbar.html index 7211e59a..8436ab6f 100644 --- a/pytest_mpl/summary/templates/navbar.html +++ b/pytest_mpl/summary/templates/navbar.html @@ -32,8 +32,7 @@ -->
- - +
+ +
+ + + + +
+
diff --git a/pytest_mpl/summary/templates/result.html b/pytest_mpl/summary/templates/result.html index 41190c60..cc71b84f 100644 --- a/pytest_mpl/summary/templates/result.html +++ b/pytest_mpl/summary/templates/result.html @@ -2,6 +2,7 @@ +
{image} {offcanvas} From 7247c58cfd3211c8701dbf6c15c5f3b63e35b644 Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Sat, 8 Jan 2022 14:46:46 +0000 Subject: [PATCH 07/24] Improve sorting order Signed-off-by: Conor MacBride --- pytest_mpl/summary/html.py | 22 +++++++++++----------- pytest_mpl/summary/templates/extra.js | 2 +- pytest_mpl/summary/templates/filter.html | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pytest_mpl/summary/html.py b/pytest_mpl/summary/html.py index 482a2843..9d29180a 100644 --- a/pytest_mpl/summary/html.py +++ b/pytest_mpl/summary/html.py @@ -42,18 +42,18 @@ def template(name): def get_status_sort(status): - s = 50 + s = 0 if status['overall'] == 'failed': - s -= 10 + s += 10 if status['image'] == 'diff': - s -= 3 + s += 3 elif status['image'] == 'missing': - s -= 4 + s += 4 if status['hash'] == 'diff': - s -= 1 + s += 1 elif status['hash'] == 'missing': - s -= 5 - return s + s += 5 + return f"{s:02.0f}" def get_status(item, card_id, warn_missing): @@ -135,13 +135,13 @@ def card(name, item, warn_missing=None): if status['image'] == 'match': rms = '< tolerance' - rms_sort = 999999 + rms_sort = "000000" elif status['image'] == 'diff': rms = item['rms'] - rms_sort = 99999 - item['rms'] + rms_sort = f"{(item['rms']+2)*1000:06.0f}" else: rms = 'None' - rms_sort = 999998 + rms_sort = "000001" offcanvas = RESULT_IMAGES.format( @@ -206,7 +206,7 @@ def generate_summary_html(results, results_dir): cards = [] for name, item in results.items(): cards += [card(name, item, warn_missing=warn_missing)] - cards = [j[0] for j in sorted(cards, key=lambda i: i[1])] + cards = [j[0] for j in sorted(cards, key=lambda i: i[1], reverse=True)] # Generate HTML html = BASE.format( diff --git a/pytest_mpl/summary/templates/extra.js b/pytest_mpl/summary/templates/extra.js index b50ccd31..94333f1c 100644 --- a/pytest_mpl/summary/templates/extra.js +++ b/pytest_mpl/summary/templates/extra.js @@ -17,7 +17,7 @@ var options = { 'rms-value', 'baseline-hash-value', 'result-hash-value'] }; var resultsList = new List('results', options); -resultsList.sort('status-sort'); +resultsList.sort('status-sort', { order: "desc" }); function applyFilters() { var cond_and = document.getElementById('filterForm').elements['conditionand'].checked; diff --git a/pytest_mpl/summary/templates/filter.html b/pytest_mpl/summary/templates/filter.html index fcfb8e75..2c5fbcac 100644 --- a/pytest_mpl/summary/templates/filter.html +++ b/pytest_mpl/summary/templates/filter.html @@ -6,13 +6,13 @@
Filter results
Sort tests by...
- - +
From cf297ec60d4a4c9a99217e59170e0aeff52dab86 Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Sat, 8 Jan 2022 15:58:42 +0000 Subject: [PATCH 08/24] Responsive search bar Signed-off-by: Conor MacBride --- pytest_mpl/summary/templates/navbar.html | 40 +++++------------------- pytest_mpl/summary/templates/styles.css | 21 +++++++++++++ 2 files changed, 28 insertions(+), 33 deletions(-) diff --git a/pytest_mpl/summary/templates/navbar.html b/pytest_mpl/summary/templates/navbar.html index 8436ab6f..b115c39a 100644 --- a/pytest_mpl/summary/templates/navbar.html +++ b/pytest_mpl/summary/templates/navbar.html @@ -1,37 +1,11 @@ -