Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 03d1f76

Browse files
ConorMacBrideConorMacBride
committed
Convert status_msg to regex
1 parent e44c0c1 commit 03d1f76

File tree

7 files changed

+139
-67
lines changed

7 files changed

+139
-67
lines changed

‎tests/subtests/helpers.py

Lines changed: 73 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
import json
33
from pathlib import Path
44

5-
__all__ = ['diff_summary', 'assert_existence', 'patch_summary']
5+
__all__ = ['diff_summary', 'assert_existence', 'patch_summary', 'apply_regex']
66

77

88
class MatchError(Exception):
@@ -31,10 +31,14 @@ def diff_summary(baseline, result, baseline_hash_library=None, result_hash_libra
3131
# Load "correct" baseline hashes
3232
with open(baseline_hash_library, 'r') as f:
3333
baseline_hash_library = json.load(f)
34+
else:
35+
baseline_hash_library = {}
3436
if result_hash_library and result_hash_library.exists():
3537
# Load "correct" result hashes
3638
with open(result_hash_library, 'r') as f:
3739
result_hash_library = json.load(f)
40+
else:
41+
result_hash_library = {}
3842

3943
# Get test names
4044
baseline_tests = set(baseline.keys())
@@ -176,3 +180,71 @@ def assert_existence(summary, items=('baseline_image', 'diff_image', 'result_ima
176180
for item in items:
177181
if test[item] is not None:
178182
assert (Path(path) / test[item]).exists()
183+
184+
185+
def _escape_regex(msg):
186+
if not msg.startswith('REGEX:'):
187+
msg = msg.replace('.', r'\.').replace('(', r'\(').replace(')', r'\)')
188+
msg = 'REGEX:' + msg
189+
return msg
190+
191+
192+
def _escape_path(msg, path):
193+
pattern = (rf"({path}[A-Za-z0-9_\-\/.\\]*)" +
194+
r"(baseline\\.png|result-failed-diff\\.png|result\\.png|\\.json)")
195+
msg = re.sub(pattern, r".*\2", msg)
196+
pattern = rf"({path}[A-Za-z0-9_\-\/.\\]*)"
197+
msg = re.sub(pattern, r".*", msg)
198+
return msg
199+
200+
201+
def _escape_float(msg, key):
202+
pattern = rf"({key}[0-9]+\\\.[0-9]{{1}})([0-9]+)"
203+
msg = re.sub(pattern, r"\1[0-9]*", msg)
204+
return msg
205+
206+
207+
def apply_regex(file, regex_paths, regex_strs):
208+
"""Convert all `status_msg` entries in JSON summary file to regex.
209+
210+
Use in your own script to assist with updating baseline summaries.
211+
212+
Parameters
213+
----------
214+
file : Path
215+
JSON summary file to convert `status_msg` to regex in. Overwritten.
216+
regex_paths : list of str
217+
List of path beginnings to identify paths that need to be converted to regex.
218+
E.g. `['/home/user/']`
219+
Does: `aaa /home/user/pytest/tmp/result\\.png bbb` -> `aaa .*result\\.png bbb`
220+
regex_strs : list of str
221+
List of keys to convert following floats to 1 d.p.
222+
E.g. ['RMS Value: ']
223+
Does: `aaa RMS Value: 12\\.432644 bbb` -> `aaa RMS Value: 12\\.4[0-9]* bbb`
224+
"""
225+
226+
with open(file, 'r') as f:
227+
summary = json.load(f)
228+
229+
for test in summary.keys():
230+
231+
msg = summary[test]['status_msg']
232+
233+
for signal in [*regex_paths, *regex_strs]:
234+
if signal in msg:
235+
msg = _escape_regex(msg)
236+
if not msg.startswith('REGEX:'):
237+
continue
238+
239+
for signal in regex_paths:
240+
if signal in msg:
241+
msg = _escape_path(msg, path=signal)
242+
243+
for signal in regex_strs:
244+
if signal in msg:
245+
msg = _escape_float(msg, key=signal)
246+
247+
summary[test]['status_msg'] = msg
248+
249+
with open(file, 'w') as f:
250+
json.dump(summary, f, indent=2)

‎tests/subtests/summaries/test_default.json

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
},
1313
"subtests.subtest.test_hmatch_idiff": {
1414
"status": "failed",
15-
"status_msg": "Error: Image files did not match.\n RMS Value: 11.100353848213828\n Expected: \n /private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hmatch_idiff/baseline.png\n Actual: \n /private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hmatch_idiff/result.png\n Difference:\n /private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hmatch_idiff/result-failed-diff.png\n Tolerance: \n 2",
15+
"status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2",
1616
"baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png",
1717
"diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png",
1818
"rms": 11.100353848213828,
@@ -23,7 +23,7 @@
2323
},
2424
"subtests.subtest.test_hmatch_imissing": {
2525
"status": "failed",
26-
"status_msg": "Image file not found for comparison test in: \n\t/Users/conor/Documents/pytest-mpl/tests/subtests/baseline\n(This is expected for new tests.)\nGenerated Image: \n\t/private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hmatch_imissing/result.png",
26+
"status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png",
2727
"baseline_image": null,
2828
"diff_image": null,
2929
"rms": null,
@@ -45,7 +45,7 @@
4545
},
4646
"subtests.subtest.test_hdiff_idiff": {
4747
"status": "failed",
48-
"status_msg": "Error: Image files did not match.\n RMS Value: 11.182677079602481\n Expected: \n /private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hdiff_idiff/baseline.png\n Actual: \n /private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hdiff_idiff/result.png\n Difference:\n /private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hdiff_idiff/result-failed-diff.png\n Tolerance: \n 2",
48+
"status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2",
4949
"baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png",
5050
"diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png",
5151
"rms": 11.182677079602481,
@@ -56,7 +56,7 @@
5656
},
5757
"subtests.subtest.test_hdiff_imissing": {
5858
"status": "failed",
59-
"status_msg": "Image file not found for comparison test in: \n\t/Users/conor/Documents/pytest-mpl/tests/subtests/baseline\n(This is expected for new tests.)\nGenerated Image: \n\t/private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hdiff_imissing/result.png",
59+
"status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png",
6060
"baseline_image": null,
6161
"diff_image": null,
6262
"rms": null,
@@ -78,7 +78,7 @@
7878
},
7979
"subtests.subtest.test_hmissing_idiff": {
8080
"status": "failed",
81-
"status_msg": "Error: Image files did not match.\n RMS Value: 12.12938597648977\n Expected: \n /private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hmissing_idiff/baseline.png\n Actual: \n /private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hmissing_idiff/result.png\n Difference:\n /private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hmissing_idiff/result-failed-diff.png\n Tolerance: \n 2",
81+
"status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2",
8282
"baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png",
8383
"diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png",
8484
"rms": 12.12938597648977,
@@ -89,7 +89,7 @@
8989
},
9090
"subtests.subtest.test_hmissing_imissing": {
9191
"status": "failed",
92-
"status_msg": "Image file not found for comparison test in: \n\t/Users/conor/Documents/pytest-mpl/tests/subtests/baseline\n(This is expected for new tests.)\nGenerated Image: \n\t/private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hmissing_imissing/result.png",
92+
"status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png",
9393
"baseline_image": null,
9494
"diff_image": null,
9595
"rms": null,
@@ -111,7 +111,7 @@
111111
},
112112
"subtests.subtest.test_hdiff_idiff_tolerance": {
113113
"status": "failed",
114-
"status_msg": "Error: Image files did not match.\n RMS Value: 29.260332173249314\n Expected: \n /private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hdiff_idiff_tolerance/baseline.png\n Actual: \n /private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hdiff_idiff_tolerance/result.png\n Difference:\n /private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png\n Tolerance: \n 3",
114+
"status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3",
115115
"baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png",
116116
"diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png",
117117
"rms": 29.260332173249314,
@@ -153,4 +153,4 @@
153153
"baseline_hash": null,
154154
"result_hash": null
155155
}
156-
}
156+
}

‎tests/subtests/summaries/test_hash.json

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
},
3535
"subtests.subtest.test_hdiff_imatch": {
3636
"status": "failed",
37-
"status_msg": "Hash 6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 doesn't match hash d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 in library /Users/conor/Documents/pytest-mpl/tests/subtests/hashes/mpl34_ft261.json for test subtests.subtest.test_hdiff_imatch.",
37+
"status_msg": "REGEX:Hash 6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 doesn't match hash d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.",
3838
"baseline_image": null,
3939
"diff_image": null,
4040
"rms": null,
@@ -45,7 +45,7 @@
4545
},
4646
"subtests.subtest.test_hdiff_idiff": {
4747
"status": "failed",
48-
"status_msg": "Hash 443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 doesn't match hash d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 in library /Users/conor/Documents/pytest-mpl/tests/subtests/hashes/mpl34_ft261.json for test subtests.subtest.test_hdiff_idiff.",
48+
"status_msg": "REGEX:Hash 443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 doesn't match hash d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff\\.",
4949
"baseline_image": null,
5050
"diff_image": null,
5151
"rms": null,
@@ -56,7 +56,7 @@
5656
},
5757
"subtests.subtest.test_hdiff_imissing": {
5858
"status": "failed",
59-
"status_msg": "Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library /Users/conor/Documents/pytest-mpl/tests/subtests/hashes/mpl34_ft261.json for test subtests.subtest.test_hdiff_imissing.",
59+
"status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.",
6060
"baseline_image": null,
6161
"diff_image": null,
6262
"rms": null,
@@ -67,7 +67,7 @@
6767
},
6868
"subtests.subtest.test_hmissing_imatch": {
6969
"status": "failed",
70-
"status_msg": "Hash for test 'subtests.subtest.test_hmissing_imatch' not found in /Users/conor/Documents/pytest-mpl/tests/subtests/hashes/mpl34_ft261.json. Generated hash is eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824.",
70+
"status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824\\.",
7171
"baseline_image": null,
7272
"diff_image": null,
7373
"rms": null,
@@ -78,7 +78,7 @@
7878
},
7979
"subtests.subtest.test_hmissing_idiff": {
8080
"status": "failed",
81-
"status_msg": "Hash for test 'subtests.subtest.test_hmissing_idiff' not found in /Users/conor/Documents/pytest-mpl/tests/subtests/hashes/mpl34_ft261.json. Generated hash is e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab.",
81+
"status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab\\.",
8282
"baseline_image": null,
8383
"diff_image": null,
8484
"rms": null,
@@ -89,7 +89,7 @@
8989
},
9090
"subtests.subtest.test_hmissing_imissing": {
9191
"status": "failed",
92-
"status_msg": "Hash for test 'subtests.subtest.test_hmissing_imissing' not found in /Users/conor/Documents/pytest-mpl/tests/subtests/hashes/mpl34_ft261.json. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b.",
92+
"status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.",
9393
"baseline_image": null,
9494
"diff_image": null,
9595
"rms": null,
@@ -100,7 +100,7 @@
100100
},
101101
"subtests.subtest.test_hdiff_imatch_tolerance": {
102102
"status": "failed",
103-
"status_msg": "Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library /Users/conor/Documents/pytest-mpl/tests/subtests/hashes/mpl34_ft261.json for test subtests.subtest.test_hdiff_imatch_tolerance.",
103+
"status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.",
104104
"baseline_image": null,
105105
"diff_image": null,
106106
"rms": null,
@@ -111,7 +111,7 @@
111111
},
112112
"subtests.subtest.test_hdiff_idiff_tolerance": {
113113
"status": "failed",
114-
"status_msg": "Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library /Users/conor/Documents/pytest-mpl/tests/subtests/hashes/mpl34_ft261.json for test subtests.subtest.test_hdiff_idiff_tolerance.",
114+
"status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.",
115115
"baseline_image": null,
116116
"diff_image": null,
117117
"rms": null,
@@ -122,7 +122,7 @@
122122
},
123123
"subtests.subtest.test_hdiff_imatch_savefig": {
124124
"status": "failed",
125-
"status_msg": "Hash 5dc1c2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 doesn't match hash d1ffc2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 in library /Users/conor/Documents/pytest-mpl/tests/subtests/hashes/mpl34_ft261.json for test subtests.subtest.test_hdiff_imatch_savefig.",
125+
"status_msg": "REGEX:Hash 5dc1c2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 doesn't match hash d1ffc2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.",
126126
"baseline_image": null,
127127
"diff_image": null,
128128
"rms": null,
@@ -133,7 +133,7 @@
133133
},
134134
"subtests.subtest.test_hdiff_imatch_style": {
135135
"status": "failed",
136-
"status_msg": "Hash 185ed1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 doesn't match hash d1ffd1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 in library /Users/conor/Documents/pytest-mpl/tests/subtests/hashes/mpl34_ft261.json for test subtests.subtest.test_hdiff_imatch_style.",
136+
"status_msg": "REGEX:Hash 185ed1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 doesn't match hash d1ffd1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.",
137137
"baseline_image": null,
138138
"diff_image": null,
139139
"rms": null,
@@ -144,7 +144,7 @@
144144
},
145145
"subtests.subtest.test_hdiff_imatch_removetext": {
146146
"status": "failed",
147-
"status_msg": "Hash be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e doesn't match hash d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e in library /Users/conor/Documents/pytest-mpl/tests/subtests/hashes/mpl34_ft261.json for test subtests.subtest.test_hdiff_imatch_removetext.",
147+
"status_msg": "REGEX:Hash be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e doesn't match hash d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.",
148148
"baseline_image": null,
149149
"diff_image": null,
150150
"rms": null,
@@ -153,4 +153,4 @@
153153
"baseline_hash": "d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e",
154154
"result_hash": "be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e"
155155
}
156-
}
156+
}

0 commit comments

Comments
 (0)