Allow to configure lazy loadable magics.

Carreau · Carreau · commit e5bbea060db0 · 2022-02-08T12:02:56.000+01:00
While we encourage load_ipython_ext to be lazy and not take too much
resources, it might not be practical especially when the top level
module takes a long time to import.

Here we allow to define a mapping between magics names and extension
name, and on attempt to execute a non-existing magics we'll look into
the lazy mapping and try to load it.

We also add a helper to let us interactively register a magic lazily,
see the `register_lazy()` method of to magics_manager
diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py
@@ -500,7 +500,6 @@ def profile(self):
     def __init__(self, ipython_dir=None, profile_dir=None,
                  user_module=None, user_ns=None,
                  custom_exceptions=((), None), **kwargs):
-
         # This is where traits with a config_key argument are updated
         # from the values on config.
         super(InteractiveShell, self).__init__(**kwargs)
@@ -1643,7 +1642,7 @@ def _inspect(self, meth, oname, namespaces=None, **kw):
                     formatter,
                     info,
                     enable_html_pager=self.enable_html_pager,
-                    **kw
+                    **kw,
                 )
             else:
                 pmethod(info.obj, oname)
@@ -2173,7 +2172,34 @@ def register_magic_function(self, func, magic_kind='line', magic_name=None):
             func, magic_kind=magic_kind, magic_name=magic_name
         )
 
-    def run_line_magic(self, magic_name, line, _stack_depth=1):
+    def _find_with_lazy_load(self, /, type_, magic_name: str):
+        """
+        Try to find a magic potentially lazy-loading it.
+
+        Parameters
+        ----------
+
+        type_: "line"|"cell"
+            the type of magics we are trying to find/lazy load.
+        magic_name: str
+            The name of the magic we are trying to find/lazy load
+
+
+        Note that this may have any side effects
+        """
+        finder = {"line": self.find_line_magic, "cell": self.find_cell_magic}[type_]
+        fn = finder(magic_name)
+        if fn is not None:
+            return fn
+        lazy = self.magics_manager.lazy_magics.get(magic_name)
+        if lazy is None:
+            return None
+
+        self.run_line_magic("load_ext", lazy)
+        res = finder(magic_name)
+        return res
+
+    def run_line_magic(self, magic_name: str, line, _stack_depth=1):
         """Execute the given line magic.
 
         Parameters
@@ -2186,7 +2212,12 @@ def run_line_magic(self, magic_name, line, _stack_depth=1):
             If run_line_magic() is called from magic() then _stack_depth=2.
             This is added to ensure backward compatibility for use of 'get_ipython().magic()'
         """
-        fn = self.find_line_magic(magic_name)
+        fn = self._find_with_lazy_load("line", magic_name)
+        if fn is None:
+            lazy = self.magics_manager.lazy_magics.get(magic_name)
+            if lazy:
+                self.run_line_magic("load_ext", lazy)
+                fn = self.find_line_magic(magic_name)
         if fn is None:
             cm = self.find_cell_magic(magic_name)
             etpl = "Line magic function `%%%s` not found%s."
@@ -2237,7 +2268,7 @@ def run_cell_magic(self, magic_name, line, cell):
         cell : str
             The body of the cell as a (possibly multiline) string.
         """
-        fn = self.find_cell_magic(magic_name)
+        fn = self._find_with_lazy_load("cell", magic_name)
         if fn is None:
             lm = self.find_line_magic(magic_name)
             etpl = "Cell magic `%%{0}` not found{1}."
diff --git a/IPython/core/magic.py b/IPython/core/magic.py
@@ -302,6 +302,34 @@ class MagicsManager(Configurable):
     # holding the actual callable object as value.  This is the dict used for
     # magic function dispatch
     magics = Dict()
+    lazy_magics = Dict(
+        help="""
+    Mapping from magic names to modules to load.
+
+    This can be used in IPython/IPykernel configuration to declare lazy magics
+    that will only be imported/registered on first use.
+
+    For example::
+
+        c.MagicsManger.lazy_magics = {
+          "my_magic": "slow.to.import",
+          "my_other_magic": "also.slow",
+        }
+
+    On first invocation of `%my_magic`, `%%my_magic`, `%%my_other_magic` or
+    `%%my_other_magic`, the corresponding module will be loaded as an ipython
+    extensions as if you had previously done `%load_ext ipython`.
+
+    Magics names should be without percent(s) as magics can be both cell
+    and line magics.
+
+    Lazy loading happen relatively late in execution process, and
+    complex extensions that manipulate Python/IPython internal state or global state
+    might not support lazy loading.
+    """
+    ).tag(
+        config=True,
+    )
 
     # A registry of the original objects that we've been given holding magics.
     registry = Dict()
@@ -366,6 +394,24 @@ def lsmagic_docs(self, brief=False, missing=''):
             docs[m_type] = m_docs
         return docs
 
+    def register_lazy(self, name: str, fully_qualified_name: str):
+        """
+        Lazily register a magic via an extension.
+
+
+        Parameters
+        ----------
+        name : str
+            Name of the magic you wish to register.
+        fully_qualified_name :
+            Fully qualified name of the module/submodule that should be loaded
+            as an extensions when the magic is first called.
+            It is assumed that loading this extensions will register the given
+            magic.
+        """
+
+        self.lazy_magics[name] = fully_qualified_name
+
     def register(self, *magic_objects):
         """Register one or more instances of Magics.
 
diff --git a/IPython/core/tests/test_magic.py b/IPython/core/tests/test_magic.py
@@ -34,9 +34,11 @@
 from IPython.utils.io import capture_output
 from IPython.utils.process import find_cmd
 from IPython.utils.tempdir import TemporaryDirectory, TemporaryWorkingDirectory
+from IPython.utils.syspathcontext import prepended_to_syspath
 
 from .test_debugger import PdbTestInput
 
+from tempfile import NamedTemporaryFile
 
 @magic.magics_class
 class DummyMagics(magic.Magics): pass
@@ -1325,13 +1327,53 @@ def test_timeit_arguments():
     _ip.magic("timeit -n1 -r1 a=('#')")
 
 
+MINIMAL_LAZY_MAGIC = """
+from IPython.core.magic import (
+    Magics,
+    magics_class,
+    line_magic,
+    cell_magic,
+)
+
+
+@magics_class
+class LazyMagics(Magics):
+    @line_magic
+    def lazy_line(self, line):
+        print("Lazy Line")
+
+    @cell_magic
+    def lazy_cell(self, line, cell):
+        print("Lazy Cell")
+
+
+def load_ipython_extension(ipython):
+    ipython.register_magics(LazyMagics)
+"""
+
+
+def test_lazy_magics():
+    with pytest.raises(UsageError):
+        ip.run_line_magic("lazy_line", "")
+
+    startdir = os.getcwd()
+
+    with TemporaryDirectory() as tmpdir:
+        with prepended_to_syspath(tmpdir):
+            ptempdir = Path(tmpdir)
+            tf = ptempdir / "lazy_magic_module.py"
+            tf.write_text(MINIMAL_LAZY_MAGIC)
+            ip.magics_manager.register_lazy("lazy_line", Path(tf.name).name[:-3])
+            with tt.AssertPrints("Lazy Line"):
+                ip.run_line_magic("lazy_line", "")
+
+
 TEST_MODULE = """
 print('Loaded my_tmp')
 if __name__ == "__main__":
     print('I just ran a script')
 """
 
-
 def test_run_module_from_import_hook():
     "Test that a module can be loaded via an import hook"
     with TemporaryDirectory() as tmpdir:
diff --git a/IPython/core/tests/test_magic_terminal.py b/IPython/core/tests/test_magic_terminal.py
@@ -9,11 +9,35 @@
 from unittest import TestCase
 
 from IPython.testing import tools as tt
-
 #-----------------------------------------------------------------------------
 # Test functions begin
 #-----------------------------------------------------------------------------
 
+
+MINIMAL_LAZY_MAGIC = """
+from IPython.core.magic import (
+    Magics,
+    magics_class,
+    line_magic,
+    cell_magic,
+)
+
+
+@magics_class
+class LazyMagics(Magics):
+    @line_magic
+    def lazy_line(self, line):
+        print("Lazy Line")
+
+    @cell_magic
+    def lazy_cell(self, line, cell):
+        print("Lazy Cell")
+
+
+def load_ipython_extension(ipython):
+    ipython.register_magics(LazyMagics)
+"""
+
 def check_cpaste(code, should_fail=False):
     """Execute code via 'cpaste' and ensure it was executed, unless
     should_fail is set.
@@ -31,7 +55,7 @@ def check_cpaste(code, should_fail=False):
     try:
         context = tt.AssertPrints if should_fail else tt.AssertNotPrints
         with context("Traceback (most recent call last)"):
-                ip.magic('cpaste')
+            ip.run_line_magic("cpaste", "")
 
         if not should_fail:
             assert ip.user_ns['code_ran'], "%r failed" % code
@@ -68,13 +92,14 @@ def runf():
         check_cpaste(code, should_fail=True)
 
 
+
 class PasteTestCase(TestCase):
     """Multiple tests for clipboard pasting"""
 
     def paste(self, txt, flags='-q'):
         """Paste input text, by default in quiet mode"""
-        ip.hooks.clipboard_get = lambda : txt
-        ip.magic('paste '+flags)
+        ip.hooks.clipboard_get = lambda: txt
+        ip.run_line_magic("paste", flags)
 
     def setUp(self):
         # Inject fake clipboard hook but save original so we can restore it later
@@ -114,7 +139,7 @@ def test_paste_py_multi_r(self):
         self.assertEqual(ip.user_ns.pop("x"), [1, 2, 3])
         self.assertEqual(ip.user_ns.pop("y"), [1, 4, 9])
         self.assertFalse("x" in ip.user_ns)
-        ip.magic("paste -r")
+        ip.run_line_magic("paste", "-r")
         self.assertEqual(ip.user_ns["x"], [1, 2, 3])
         self.assertEqual(ip.user_ns["y"], [1, 4, 9])
 
diff --git a/IPython/terminal/ipapp.py b/IPython/terminal/ipapp.py
@@ -25,6 +25,7 @@
 from IPython.core.application import (
     ProfileDir, BaseIPythonApplication, base_flags, base_aliases
 )
+from IPython.core.magic import MagicsManager
 from IPython.core.magics import (
     ScriptMagics, LoggingMagics
 )
@@ -200,6 +201,7 @@ def _classes_default(self):
             self.__class__,      # it will also affect subclasses (e.g. QtConsole)
             TerminalInteractiveShell,
             HistoryManager,
+            MagicsManager,
             ProfileDir,
             PlainTextFormatter,
             IPCompleter,
diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst
@@ -3,6 +3,24 @@
 ============
 
 
+.. _version 7.32:
+
+IPython 7.32
+============
+
+
+The ability to configure magics to be lazily loaded has been added to IPython.
+See the ``ipython --help-all`` section on ``MagicsManager.lazy_magic``.
+One can now use::
+
+    c.MagicsManger.lazy_magics = {
+              "my_magic": "slow.to.import",
+              "my_other_magic": "also.slow",
+    }
+
+And on first use of ``%my_magic``, or corresponding cell magic, or other line magic,
+the corresponding ``load_ext`` will be called just before trying to invoke the magic.
+
 .. _version 7.31:
 
 IPython 7.31
