Deduplicate completions between IPython and Jedi.

Carreau · Carreau · commit cd7ef695778b · 2017-02-11T22:04:24.000-08:00
Completions coming from different sources may be in practice identical, but the object themselves differ. Provide a function to properly deduplicate them. Not perfect for #10282 in particular with trailing spaces, but can be dealt with later.
diff --git a/IPython/core/completer.py b/IPython/core/completer.py
@@ -344,11 +344,52 @@ def __eq__(self, other)->Bool:
     def __hash__(self):
         return hash((self.start, self.end, self.text))
 
+
 _IC = Iterator[Completion]
 
-def rectify_completions(text:str, completion:_IC, *, _debug=False)->_IC:
+
+def _deduplicate_completions(text: str, completions: _IC)-> _IC:
     """
-    Rectify a set of completion to all have the same ``start`` and ``end``
+    Deduplicate a set of completions.
+
+    .. warning:: Unstable
+
+        This function is unstable, API may change without warning.
+
+    Parameters
+    ----------
+    text: str
+        text that should be completed.
+    completions: Iterator[Completion]
+        iterator over the completions to deduplicate
+
+
+    Completions coming from multiple sources, may be different but end up having
+    the same effect when applied to ``text``. If this is the case, this will
+    consider completions as equal and only emit the first encountered.
+
+    Not folded in `completions()` yet for debugging purpose, and to detect when
+    the IPython completer does return things that Jedi does not, but should be
+    at some point.
+    """
+    completions = list(completions)
+    if not completions:
+        return
+
+    new_start = min(c.start for c in completions)
+    new_end = max(c.end for c in completions)
+
+    seen = set()
+    for c in completions:
+        new_text = text[new_start:c.start] + c.text + text[c.end:new_end]
+        if new_text not in seen:
+            yield c
+            seen.add(new_text)
+
+
+def rectify_completions(text: str, completions: _IC, *, _debug=False)->_IC:
+    """
+    Rectify a set of completions to all have the same ``start`` and ``end``
 
     .. warning:: Unstable
 
@@ -359,7 +400,7 @@ def rectify_completions(text:str, completion:_IC, *, _debug=False)->_IC:
     ----------
     text: str
         text that should be completed.
-    completion: Iterator[Completion]
+    completions: Iterator[Completion]
         iterator over the completions to rectify
 
 
@@ -1510,22 +1551,17 @@ def completions(self, text: str, offset: int)->Iterator[Completion]:
             fake Completion token to distinguish completion returned by Jedi
             and usual IPython completion.
 
+        .. note::
+            
+            Completions are not completely deduplicated yet. If identical
+            completions are coming from different sources this function does not
+            ensure that each completion object will only be present once.
         """
         warnings.warn("_complete is a provisional API (as of IPython 6.0). "
                       "It may change without warnings. "
                       "Use in corresponding context manager.",
                       category=ProvisionalCompleterWarning, stacklevel=2)
 
-        # Possible Improvements / Known limitation
-        ##########################################
-        # Completions may be identical even if they have different ranges and
-        # text. For example:
-        # >>> a=1
-        # >>> a.<tab>
-        # May returns:
-        #  - `a.real` from 0 to 2
-        #  - `.real` from 1 to 2
-        # the current code does not (yet) check for such equivalence
         seen = set()
         for c in self._completions(text, offset, _timeout=self.jedi_compute_type_timeout/1000):
             if c and (c in seen):
diff --git a/IPython/core/tests/test_completer.py b/IPython/core/tests/test_completer.py
@@ -6,6 +6,7 @@
 
 import os
 import sys
+import textwrap
 import unittest
 
 from contextlib import contextmanager
@@ -20,7 +21,8 @@
 from IPython.utils.generics import complete_object
 from IPython.testing import decorators as dec
 
-from IPython.core.completer import Completion, provisionalcompleter, match_dict_keys
+from IPython.core.completer import (
+    Completion, provisionalcompleter, match_dict_keys, _deduplicate_completions)
 from nose.tools import assert_in, assert_not_in
 
 #-----------------------------------------------------------------------------
@@ -294,14 +296,29 @@ def _test_not_complete(reason, s, comp):
 
     import jedi
     jedi_version = tuple(int(i) for i in jedi.__version__.split('.')[:3])
-    if jedi_version > (0,10):
+    if jedi_version > (0, 10):
         yield _test_complete, 'jedi >0.9 should complete and not crash', 'a=1;a.', 'real'
     yield _test_complete, 'can infer first argument', 'a=(1,"foo");a[0].', 'real'
     yield _test_complete, 'can infer second argument', 'a=(1,"foo");a[1].', 'capitalize'
     yield _test_complete, 'cover duplicate completions', 'im', 'import', 0, 2
 
     yield _test_not_complete, 'does not mix types', 'a=(1,"foo");a[0].', 'capitalize'
 
+def test_deduplicate_completions():
+    """
+    Test that completions are correctly deduplicated (even if ranges are not the same)
+    """
+    ip = get_ipython()
+    ip.ex(textwrap.dedent('''
+    class Z:
+        zoo = 1
+    '''))
+    with provisionalcompleter():
+        l = list(_deduplicate_completions('Z.z', ip.Completer.completions('Z.z', 3)))
+
+    assert len(l) == 1, 'Completions (Z.z<tab>) correctly deduplicate: %s ' % l
+    assert l[0].text == 'zoo'  # and not `it.accumulate`
+
 
 def test_greedy_completions():
     """
diff --git a/IPython/terminal/ptutils.py b/IPython/terminal/ptutils.py
@@ -10,7 +10,9 @@
 import unicodedata
 from wcwidth import wcwidth
 
-from IPython.core.completer import IPCompleter, provisionalcompleter, rectify_completions, cursor_to_position
+from IPython.core.completer import (
+    IPCompleter, provisionalcompleter, rectify_completions, cursor_to_position,
+    _deduplicate_completions)
 from prompt_toolkit.completion import Completer, Completion
 from prompt_toolkit.layout.lexers import Lexer
 from prompt_toolkit.layout.lexers import PygmentsLexer
@@ -61,8 +63,8 @@ def _get_completions(body, offset, cursor_position, ipyc):
         Private equivalent of get_completions() use only for unit_testing.
         """
         debug = getattr(ipyc, 'debug', False)
-        completions = rectify_completions(
-            body, ipyc.completions(body, offset), _debug=debug)
+        completions = _deduplicate_completions(
+            body, ipyc.completions(body, offset))
         for c in completions:
             if not c.text:
                 # Guard against completion machinery giving us an empty string.
