From d80876d8fa63140ab20dfa0674ba62caf67c580a Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Fri, 17 Sep 2021 17:28:47 +0200 Subject: [PATCH 1/7] enh: cast lists into tuples when printing inputs diffs --- nipype/utils/misc.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/nipype/utils/misc.py b/nipype/utils/misc.py index 8ec6ee5342..97b0f4fe0c 100644 --- a/nipype/utils/misc.py +++ b/nipype/utils/misc.py @@ -325,17 +325,17 @@ def dict_diff(dold, dnew, indent=0): # so we need to join the messages together for k in new_keys.intersection(old_keys): try: - new, old = dnew[k], dold[k] - same = new == old - if not same: - # Since JSON does not discriminate between lists and - # tuples, we might need to cast them into the same type - # as the last resort. And lets try to be more generic - same = old.__class__(new) == old + # Reading from JSON produces lists, but internally we typically + # use tuples. At this point these dictionary values can be + # immutable (and therefore the preference for tuple). + new = tuple([tuple(el) if isinstance(el, list) else el for el in dnew[k]]) + old = tuple([tuple(el) if isinstance(el, list) else el for el in dold[k]]) except Exception: - same = False - if not same: - diff += [" * %s: %r != %r" % (k, dnew[k], dold[k])] + new = dnew[k] + old = dold[k] + + if new != old: + diff += [" * %s: %r != %r" % (k, new, old)] if len(diff) > diffkeys: diff.insert(diffkeys, "Some dictionary entries had differing values:") From 44fc17206a7d3b2fb4d10a98329fc1253a5da960 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 23 Sep 2021 12:38:23 +0200 Subject: [PATCH 2/7] fix: correctly deal with dictionaries, insert ellipsis for very long diffs --- nipype/utils/misc.py | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/nipype/utils/misc.py b/nipype/utils/misc.py index 97b0f4fe0c..16638f0240 100644 --- a/nipype/utils/misc.py +++ b/nipype/utils/misc.py @@ -296,12 +296,14 @@ def dict_diff(dold, dnew, indent=0): typical use -- log difference for hashed_inputs """ - # First check inputs, since they usually are lists of tuples - # and dicts are required. - if isinstance(dnew, list): + try: dnew = dict(dnew) - if isinstance(dold, list): dold = dict(dold) + except Exception: + return textwrap_indent(f"""\ +Diff between nipype inputs failed: +* Cached inputs: {dold} +* New inputs: {dnew}""") # Compare against hashed_inputs # Keys: should rarely differ @@ -321,21 +323,30 @@ def dict_diff(dold, dnew, indent=0): diffkeys = len(diff) + def _shorten(value): + if isinstance(value, str) and len(value) > 50: + return f"{value[:10]}...{value[-10:]}" + if isinstance(value, (tuple, list)) and len(value) > 10: + return tuple( + list(value[:2]) + "..." + list(value[-2:]) + ) + return value + # Values in common keys would differ quite often, # so we need to join the messages together for k in new_keys.intersection(old_keys): - try: - # Reading from JSON produces lists, but internally we typically - # use tuples. At this point these dictionary values can be - # immutable (and therefore the preference for tuple). + new = dnew[k] + old = dold[k] + # Reading from JSON produces lists, but internally we typically + # use tuples. At this point these dictionary values can be + # immutable (and therefore the preference for tuple). + if isinstance(dnew[k], (list, tuple)): new = tuple([tuple(el) if isinstance(el, list) else el for el in dnew[k]]) + if isinstance(dnew[k], (list, tuple)): old = tuple([tuple(el) if isinstance(el, list) else el for el in dold[k]]) - except Exception: - new = dnew[k] - old = dold[k] if new != old: - diff += [" * %s: %r != %r" % (k, new, old)] + diff += [" * %s: %r != %r" % (k, _shorten(new), _shorten(old))] if len(diff) > diffkeys: diff.insert(diffkeys, "Some dictionary entries had differing values:") From f9588b63c81a6dd814b19174dd0c2ce57e23696b Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 23 Sep 2021 12:41:38 +0200 Subject: [PATCH 3/7] sty: fix some style errors --- nipype/utils/misc.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nipype/utils/misc.py b/nipype/utils/misc.py index 16638f0240..109a476909 100644 --- a/nipype/utils/misc.py +++ b/nipype/utils/misc.py @@ -300,10 +300,12 @@ def dict_diff(dold, dnew, indent=0): dnew = dict(dnew) dold = dict(dold) except Exception: - return textwrap_indent(f"""\ + return textwrap_indent( + f"""\ Diff between nipype inputs failed: * Cached inputs: {dold} -* New inputs: {dnew}""") +* New inputs: {dnew}""" + ) # Compare against hashed_inputs # Keys: should rarely differ @@ -327,9 +329,7 @@ def _shorten(value): if isinstance(value, str) and len(value) > 50: return f"{value[:10]}...{value[-10:]}" if isinstance(value, (tuple, list)) and len(value) > 10: - return tuple( - list(value[:2]) + "..." + list(value[-2:]) - ) + return tuple(list(value[:2]) + "..." + list(value[-2:])) return value # Values in common keys would differ quite often, From 871cf05b984b5dc92ce5660a19b217fa3ede4f2b Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Fri, 1 Oct 2021 10:01:07 +0200 Subject: [PATCH 4/7] Apply suggestions from code review Co-authored-by: Chris Markiewicz --- nipype/utils/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/utils/misc.py b/nipype/utils/misc.py index 109a476909..eeb6699f8a 100644 --- a/nipype/utils/misc.py +++ b/nipype/utils/misc.py @@ -329,7 +329,7 @@ def _shorten(value): if isinstance(value, str) and len(value) > 50: return f"{value[:10]}...{value[-10:]}" if isinstance(value, (tuple, list)) and len(value) > 10: - return tuple(list(value[:2]) + "..." + list(value[-2:])) + return tuple(list(value[:2]) + ["..."] + list(value[-2:])) return value # Values in common keys would differ quite often, From b80d86e36e8412dbdffab55e322b41392d4cabed Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Fri, 1 Oct 2021 10:09:22 +0200 Subject: [PATCH 5/7] enh: apply comments from review Co-authored-by: Chris Markiewicz --- nipype/utils/misc.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/nipype/utils/misc.py b/nipype/utils/misc.py index eeb6699f8a..11540ed113 100644 --- a/nipype/utils/misc.py +++ b/nipype/utils/misc.py @@ -297,8 +297,7 @@ def dict_diff(dold, dnew, indent=0): typical use -- log difference for hashed_inputs """ try: - dnew = dict(dnew) - dold = dict(dold) + dnew, dold = dict(dnew), dict(dold) except Exception: return textwrap_indent( f"""\ @@ -332,18 +331,21 @@ def _shorten(value): return tuple(list(value[:2]) + ["..."] + list(value[-2:])) return value + def _uniformize(val): + if isinstance(val, dict): + return {k: _uniformize(v) for k, v in val.items()} + if isinstance(val, (list, tuple)): + return tuple(_uniformize(el) for el in val) + return val + # Values in common keys would differ quite often, # so we need to join the messages together for k in new_keys.intersection(old_keys): - new = dnew[k] - old = dold[k] # Reading from JSON produces lists, but internally we typically # use tuples. At this point these dictionary values can be # immutable (and therefore the preference for tuple). - if isinstance(dnew[k], (list, tuple)): - new = tuple([tuple(el) if isinstance(el, list) else el for el in dnew[k]]) - if isinstance(dnew[k], (list, tuple)): - old = tuple([tuple(el) if isinstance(el, list) else el for el in dold[k]]) + new = _uniformize(dnew[k]) + old = _uniformize(dold[k]) if new != old: diff += [" * %s: %r != %r" % (k, _shorten(new), _shorten(old))] From 96b1e67f6788f9118b71370a0b0e912524485705 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Wed, 13 Oct 2021 15:06:07 -0400 Subject: [PATCH 6/7] TEST: Thoroughly test dict_diff() --- nipype/utils/tests/test_misc.py | 48 ++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/nipype/utils/tests/test_misc.py b/nipype/utils/tests/test_misc.py index ad25c6ba14..13ae3740d6 100644 --- a/nipype/utils/tests/test_misc.py +++ b/nipype/utils/tests/test_misc.py @@ -6,7 +6,13 @@ import pytest -from nipype.utils.misc import container_to_string, str2bool, flatten, unflatten +from nipype.utils.misc import ( + container_to_string, + str2bool, + flatten, + unflatten, + dict_diff, +) def test_cont_to_str(): @@ -95,3 +101,43 @@ def test_rgetcwd(monkeypatch, tmpdir): monkeypatch.delenv("PWD") with pytest.raises(OSError): rgetcwd(error=False) + + +def test_dict_diff(): + abtuple = [("a", "b")] + abdict = dict(abtuple) + + # Unchanged + assert dict_diff(abdict, abdict) == "" + assert dict_diff(abdict, abtuple) == "" + assert dict_diff(abtuple, abdict) == "" + assert dict_diff(abtuple, abtuple) == "" + + # Changed keys + diff = dict_diff({"a": "b"}, {"b": "a"}) + assert "Dictionaries had differing keys" in diff + assert "keys not previously seen: {'b'}" in diff + assert "keys not presently seen: {'a'}" in diff + + # Trigger recursive uniformization + complicated_val1 = [{"a": ["b"], "c": ("d", "e")}] + complicated_val2 = [{"a": ["x"], "c": ("d", "e")}] + uniformized_val1 = ({"a": ("b",), "c": ("d", "e")},) + uniformized_val2 = ({"a": ("x",), "c": ("d", "e")},) + + diff = dict_diff({"a": complicated_val1}, {"a": complicated_val2}) + assert "Some dictionary entries had differing values:" in diff + assert "a: {!r} != {!r}".format(uniformized_val2, uniformized_val1) in diff + + # Trigger shortening + diff = dict_diff({"a": "b" * 60}, {"a": "c" * 70}) + assert "Some dictionary entries had differing values:" in diff + assert "a: 'cccccccccc...cccccccccc' != 'bbbbbbbbbb...bbbbbbbbbb'" in diff + + # Fail the dict conversion + diff = dict_diff({}, "not a dict") + assert diff == ( + "Diff between nipype inputs failed:\n" + "* Cached inputs: {}\n" + "* New inputs: not a dict" + ) From 1beec5916daf64d2add066da68858acb58980da9 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Wed, 13 Oct 2021 15:06:34 -0400 Subject: [PATCH 7/7] FIX: Correct call of indent; drop compatibility shim --- nipype/utils/misc.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/nipype/utils/misc.py b/nipype/utils/misc.py index 11540ed113..6b106da952 100644 --- a/nipype/utils/misc.py +++ b/nipype/utils/misc.py @@ -13,16 +13,7 @@ import numpy as np -try: - from textwrap import indent as textwrap_indent -except ImportError: - - def textwrap_indent(text, prefix): - """A textwrap.indent replacement for Python < 3.3""" - if not prefix: - return text - splittext = text.splitlines(True) - return prefix + prefix.join(splittext) +import textwrap def human_order_sorted(l): @@ -299,11 +290,12 @@ def dict_diff(dold, dnew, indent=0): try: dnew, dold = dict(dnew), dict(dold) except Exception: - return textwrap_indent( + return textwrap.indent( f"""\ Diff between nipype inputs failed: * Cached inputs: {dold} -* New inputs: {dnew}""" +* New inputs: {dnew}""", + " " * indent, ) # Compare against hashed_inputs @@ -353,7 +345,7 @@ def _uniformize(val): if len(diff) > diffkeys: diff.insert(diffkeys, "Some dictionary entries had differing values:") - return textwrap_indent("\n".join(diff), " " * indent) + return textwrap.indent("\n".join(diff), " " * indent) def rgetcwd(error=True):