Don't double-decode input, causing non-UTF-8 files to be corrupted (psf#4964)

cobaltt7 · MeGaGiGaGon · web-flow · commit fe875c0ed341 · 2026-01-23T10:05:02.000-06:00
Co-authored-by: MeGaGiGaGon &lt;107241144+MeGaGiGaGon@users.noreply.github.com&gt;
diff --git a/CHANGES.md b/CHANGES.md
@@ -13,6 +13,8 @@
 
 <!-- Changes that affect Black's stable style -->
 
+- Don't double-decode input, causing non-UTF-8 files to be corrupted (#4964)
+
 ### Preview style
 
 <!-- Changes that affect Black's preview style -->
diff --git a/src/black/__init__.py b/src/black/__init__.py
@@ -1218,8 +1218,10 @@ def f(
 def _format_str_once(
     src_contents: str, *, mode: Mode, lines: Collection[tuple[int, int]] = ()
 ) -> str:
+    # Use the encoding overwrite since the src_contents may contain a different
+    # magic encoding comment than utf-8
     normalized_contents, _, newline_type = decode_bytes(
-        src_contents.encode("utf-8"), mode
+        src_contents.encode("utf-8"), mode, encoding_overwrite="utf-8"
     )
 
     src_node = lib2to3_parse(
@@ -1276,14 +1278,25 @@ def _format_str_once(
     return "".join(dst_contents).replace("\n", newline_type)
 
 
-def decode_bytes(src: bytes, mode: Mode) -> tuple[FileContent, Encoding, NewLine]:
+def decode_bytes(
+    src: bytes, mode: Mode, *, encoding_overwrite: str | None = None
+) -> tuple[FileContent, Encoding, NewLine]:
     """Return a tuple of (decoded_contents, encoding, newline).
 
-    `newline` is either CRLF or LF but `decoded_contents` is decoded with
+    `newline` is either CRLF, LF, or CR, but `decoded_contents` is decoded with
     universal newlines (i.e. only contains LF).
+
+    Use the keyword only encoding_overwrite argument if the bytes are encoded
+    differently to their possible encoding magic comment.
     """
     srcbuf = io.BytesIO(src)
+
+    # Still use detect encoding even if overrite set because otherwise lines
+    # might be different
     encoding, lines = tokenize.detect_encoding(srcbuf.readline)
+    if encoding_overwrite is not None:
+        encoding = encoding_overwrite
+
     if not lines:
         return "", encoding, "\n"
 
diff --git a/tests/test_black.py b/tests/test_black.py
@@ -19,7 +19,7 @@
 from io import BytesIO
 from pathlib import Path, WindowsPath
 from platform import system
-from tempfile import TemporaryDirectory
+from tempfile import NamedTemporaryFile, TemporaryDirectory
 from typing import Any, TypeVar
 from unittest.mock import MagicMock, patch
 
@@ -2080,12 +2080,39 @@ def test_carriage_return_edge_cases(self) -> None:
             == "class A: ...\r"
         )
 
-    def test_preview_newline_type_detection(self) -> None:
+    def test_newline_type_detection(self) -> None:
         mode = Mode()
         newline_types = ["A\n", "A\r\n", "A\r"]
         for test_case in itertools.permutations(newline_types):
             assert black.format_str("".join(test_case), mode=mode) == test_case[0] * 3
 
+    def test_decode_with_encoding(self) -> None:
+        # This uses temporary files since some editors (including GitHub)
+        # struggle with displaying and/or editing non utf-8 data
+        # \xfc is iso-8859-1 for ü
+        with NamedTemporaryFile(delete=False) as first_line:
+            first_line.write(
+                b"# -*- coding: iso-8859-1 -*-\n"
+                b"# 2002-11-22 J\xfcrgen Hermann <jh@web.de>\n"
+            )
+            first_line.close()
+            self.assertFalse(
+                ff(Path(first_line.name)),
+                "Failed to properly detect encoding",
+            )
+
+        with NamedTemporaryFile(delete=False) as second_line:
+            second_line.write(
+                b"#! /usr/bin/env python3\n"
+                b"# -*- coding: iso-8859-1 -*-\n"
+                b"# 2002-11-22 J\xfcrgen Hermann <jh@web.de>\n"
+            )
+            second_line.close()
+            self.assertFalse(
+                ff(Path(second_line.name)),
+                "Failed to properly detect encoding on second line",
+            )
+
 
 class TestCaching:
     def test_get_cache_dir(
