Implement support for image/png format in terminal (#15184)

Carreau · web-flow · commit be4179d1ba8b · 2026-04-20T10:35:05.000+02:00
Closes #13287 Now `PIL.Image` and `sympy.init_printing(use_latex="png")` show images. The Kitty graphics protocol is used as [suggested](#10610 (comment)). Alternative implementations: * Putting a [mime renderer extension](https://ipython.readthedocs.io/en/stable/config/shell_mimerenderer.html#mime-renderer-extensions) in `ipython_config.py` is inconvenient, and someone requested a [native](#10610 (comment)) feature * Probing for Kitty graphics support would detect any compatible terminal, but hardcoding terminal executable names avoids startup slowdown * Adding tmux support via Unicode-based Kitty graphics would be very complicated Alternatives protocols: * iTerm2 graphics don't work in Ghostty or st * Sixel requires complicated rasterization and dithering
diff --git a/IPython/core/kitty.py b/IPython/core/kitty.py
@@ -0,0 +1,73 @@
+# Implements https://sw.kovidgoyal.net/kitty/graphics-protocol/
+
+from base64 import b64encode, b64decode
+import sys
+from typing import Union
+
+def _supports_kitty_graphics() -> bool:
+    import platform
+
+    if platform.system() not in ("Darwin", "Linux"):
+        return False
+
+    if not sys.stdout.isatty():
+        return False
+    # Hardcoding process names instead of using
+    # https://sw.kovidgoyal.net/kitty/graphics-protocol/#querying-support-and-available-transmission-mediums
+    # to avoid startup slowdown
+    supported_terminals = {
+        "ghostty",
+        "iTerm2",
+        "kitty",
+        "konsole",
+        "warp",
+        "wayst",
+        "wezterm-gui",
+    }
+    import psutil
+
+    process = psutil.Process()
+    while process := process.parent():
+        if process.name() in supported_terminals:
+            return True
+    return False
+
+
+supports_kitty_graphics = _supports_kitty_graphics()
+
+
+def png_to_kitty_ansi(png: bytes) -> str:
+    if not png.startswith(b"\x89PNG\r\n\x1a\n"):
+        raise ValueError
+    # This simplicity resembles
+    # https://sw.kovidgoyal.net/kitty/graphics-protocol/#a-minimal-example
+    # but if we need tmux support, we can switch to Unicode like
+    # https://github.com/hzeller/timg/blob/main/src/kitty-canvas.cc
+    result = ["\033_Ga=T,f=100,", "m=1;"]
+    encoded = b64encode(png)
+    for i in range(0, len(encoded), 4096):
+        result.append(encoded[i : i + 4096].decode("ascii"))
+        result.append("\033\\")
+        result.append("\033_G")
+        result.append("m=1;")
+    del result[-2:]
+    result[-3] = "m=0;"
+    return "".join(result)
+
+
+def kitty_png_render(png: Union[bytes, str], _md_dict: object) -> None:
+    if isinstance(png, str):
+        png = png_to_kitty_ansi(b64decode(png))
+    else:
+        png = png_to_kitty_ansi(png)
+    print(png)
+
+
+display_formatter_default_active_types = [
+    "text/plain",
+    *(["image/png"] if supports_kitty_graphics else []),
+]
+
+terminal_default_mime_renderers = {
+    "image/png": kitty_png_render,
+}
diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py
@@ -8,6 +8,10 @@
 
 from IPython.core.async_helpers import get_asyncio_loop
 from IPython.core.interactiveshell import InteractiveShell, InteractiveShellABC
+from IPython.core.kitty import (
+    display_formatter_default_active_types,
+    terminal_default_mime_renderers,
+)
 from IPython.utils.py3compat import input
 from IPython.utils.PyColorize import theme_table
 from IPython.utils.terminal import toggle_set_term_title, set_term_title, restore_term_title
@@ -771,7 +775,12 @@ def init_display_formatter(self):
             "DisplayFormatter" in config
             and "active_types" in config["DisplayFormatter"]
         ):
-            self.display_formatter.active_types = ["text/plain"]
+            self.display_formatter.active_types = display_formatter_default_active_types
+            if not (
+                "TerminalInteractiveShell" in config
+                and "mime_renderers" in config["TerminalInteractiveShell"]
+            ):
+                self.mime_renderers = terminal_default_mime_renderers
 
     def init_prompt_toolkit_cli(self):
         if self.simple_prompt:
diff --git a/pyproject.toml b/pyproject.toml
@@ -30,6 +30,7 @@ dependencies = [
     "matplotlib-inline>=0.1.6",
     'pexpect>4.6; sys_platform != "win32" and sys_platform != "emscripten"',
     "prompt_toolkit>=3.0.41,<3.1.0",
+    "psutil>=7",
     "pygments>=2.14.0", # wheel
     "stack_data>=0.6.0",
     "traitlets>=5.13.0",
diff --git a/tests/test_capture.py b/tests/test_capture.py
@@ -160,7 +160,9 @@ def test_capture_output_no_stderr():
 
 def test_capture_output_no_display():
     """test capture_output(display=False)"""
-    rich = capture.RichOutput(data=full_data)
+    data = full_data.copy()
+    del data["image/png"]
+    rich = capture.RichOutput(data=data)
     with capture.capture_output(display=False) as cap:
         print(hello_stdout, end="")
         print(hello_stderr, end="", file=sys.stderr)
