cli.console: keep unicode chars in JSON output

bastimeyer · bastimeyer · commit 261680cc66fa · 2024-07-13T14:28:17.000+02:00
diff --git a/src/streamlink_cli/console.py b/src/streamlink_cli/console.py
@@ -79,7 +79,10 @@ def msg_json(self, *objs: Any, **keywords: Any) -> None:
                 out.update(**obj)
             out.update(**keywords)
 
-        msg = dumps(out, cls=JSONEncoder, indent=2)
+        # don't escape Unicode characters outside the ASCII range if the output encoding is UTF-8
+        ensure_ascii = self.output.encoding != "utf-8"
+
+        msg = dumps(out, cls=JSONEncoder, ensure_ascii=ensure_ascii, indent=2)
         self.output.write(f"{msg}\n")
 
 
diff --git a/tests/cli/test_console.py b/tests/cli/test_console.py
@@ -1,4 +1,4 @@
-from io import StringIO
+from io import BytesIO, TextIOWrapper
 from textwrap import dedent
 from unittest.mock import Mock
 
@@ -7,46 +7,80 @@
 from streamlink_cli.console import ConsoleOutput
 
 
+def getvalue(output: TextIOWrapper, size: int = -1):
+    output.seek(0)
+
+    return output.read(size)
+
+
 class TestConsoleOutput:
     @pytest.fixture(autouse=True)
     def _isatty(self, request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch):
         isatty = not request.function.__name__.endswith("_no_tty")
         monkeypatch.setattr("sys.stdin.isatty", lambda: isatty)
 
-    def test_msg(self):
-        output = StringIO()
+    @pytest.fixture()
+    def output(self, request: pytest.FixtureRequest):
+        params = getattr(request, "param", {})
+        params.setdefault("encoding", "utf-8")
+        params.setdefault("errors", "backslashreplace")
+        output = TextIOWrapper(BytesIO(), **params)
+
+        return output
+
+    @pytest.mark.parametrize(("output", "expected"), [
+        pytest.param(
+            {"encoding": "utf-8"},
+            "Bär: 🐻",
+            id="utf-8 encoding",
+        ),
+        pytest.param(
+            {"encoding": "ascii"},
+            "B\\xe4r: \\U0001f43b",  # Unicode character: "Bear Face" (U+1F43B)
+            id="ascii encoding",
+        ),
+    ], indirect=["output"])
+    def test_msg(self, output: TextIOWrapper, expected: str):
         console = ConsoleOutput(output)
-        console.msg("foo")
+        console.msg("Bär: 🐻")
         console.msg_json({"test": 1})
-        assert output.getvalue() == "foo\n"
-
-    def test_msg_json(self):
-        output = StringIO()
+        assert getvalue(output) == f"{expected}\n"
+
+    @pytest.mark.parametrize(("output", "expected"), [
+        pytest.param(
+            {"encoding": "utf-8"},
+            "Bär: 🐻",
+            id="utf-8 encoding",
+        ),
+        pytest.param(
+            {"encoding": "ascii"},
+            "B\\u00e4r: \\ud83d\\udc3b",  # Unicode character: "Bear Face" (U+1F43B) - UTF-16: 0xD83D 0xDC3B
+            id="ascii encoding",
+        ),
+    ], indirect=["output"])
+    def test_msg_json(self, output: TextIOWrapper, expected: str):
         console = ConsoleOutput(output, json=True)
         console.msg("foo")
-        console.msg_json({"test": 1})
-        assert output.getvalue() == '{\n  "test": 1\n}\n'
+        console.msg_json({"test": "Bär: 🐻"})
+        assert getvalue(output) == f'{{\n  "test": "{expected}"\n}}\n'
 
-    def test_msg_json_object(self):
-        output = StringIO()
+    def test_msg_json_object(self, output: TextIOWrapper):
         console = ConsoleOutput(output, json=True)
-        console.msg_json(Mock(__json__=Mock(return_value={"test": 1})))
-        assert output.getvalue() == '{\n  "test": 1\n}\n'
+        console.msg_json(Mock(__json__=lambda: {"test": "Hello world, Γειά σου Κόσμε, こんにちは世界"}))  # noqa: RUF001
+        assert getvalue(output) == '{\n  "test": "Hello world, Γειά σου Κόσμε, こんにちは世界"\n}\n'  # noqa: RUF001
 
-    def test_msg_json_list(self):
-        output = StringIO()
+    def test_msg_json_list(self, output: TextIOWrapper):
         console = ConsoleOutput(output, json=True)
-        test_list = ["foo", "bar"]
+        test_list = ["Hello world, Γειά σου Κόσμε, こんにちは世界", '"🐻"']  # noqa: RUF001
         console.msg_json(test_list)
-        assert output.getvalue() == '[\n  "foo",\n  "bar"\n]\n'
+        assert getvalue(output) == '[\n  "Hello world, Γειά σου Κόσμε, こんにちは世界",\n  "\\"🐻\\""\n]\n'  # noqa: RUF001
 
-    def test_msg_json_merge_object(self):
-        output = StringIO()
+    def test_msg_json_merge_object(self, output: TextIOWrapper):
         console = ConsoleOutput(output, json=True)
         test_obj1 = {"test": 1, "foo": "foo"}
         test_obj2 = Mock(__json__=Mock(return_value={"test": 2}))
         console.msg_json(test_obj1, test_obj2, ["qux"], foo="bar", baz="qux")
-        assert output.getvalue() == dedent("""
+        assert getvalue(output) == dedent("""
             {
               "test": 2,
               "foo": "bar",
@@ -55,13 +89,12 @@ def test_msg_json_merge_object(self):
         """).lstrip()
         assert list(test_obj1.items()) == [("test", 1), ("foo", "foo")]
 
-    def test_msg_json_merge_list(self):
-        output = StringIO()
+    def test_msg_json_merge_list(self, output: TextIOWrapper):
         console = ConsoleOutput(output, json=True)
         test_list1 = ["foo", "bar"]
         test_list2 = Mock(__json__=Mock(return_value={"foo": "bar"}))
         console.msg_json(test_list1, ["baz"], test_list2, {"foo": "bar"}, foo="bar", baz="qux")
-        assert output.getvalue() == dedent("""
+        assert getvalue(output) == dedent("""
             [
               "foo",
               "bar",
@@ -80,45 +113,40 @@ def test_msg_json_merge_list(self):
         """).lstrip()
         assert test_list1 == ["foo", "bar"]
 
-    def test_ask(self, monkeypatch: pytest.MonkeyPatch):
+    def test_ask(self, monkeypatch: pytest.MonkeyPatch, output: TextIOWrapper):
         monkeypatch.setattr("builtins.input", Mock(return_value="hello"))
 
-        output = StringIO()
         console = ConsoleOutput(output)
         assert console.ask("test: ") == "hello"
-        assert output.getvalue() == "test: "
+        assert getvalue(output) == "test: "
 
-    def test_ask_no_tty(self, monkeypatch: pytest.MonkeyPatch):
+    def test_ask_no_tty(self, monkeypatch: pytest.MonkeyPatch, output: TextIOWrapper):
         mock_input = Mock()
         monkeypatch.setattr("builtins.input", mock_input)
 
-        output = StringIO()
         console = ConsoleOutput(output)
         assert console.ask("test: ") is None
-        assert output.getvalue() == ""
+        assert getvalue(output) == ""
         assert mock_input.call_args_list == []
 
-    def test_ask_input_exception(self, monkeypatch: pytest.MonkeyPatch):
+    def test_ask_input_exception(self, monkeypatch: pytest.MonkeyPatch, output: TextIOWrapper):
         monkeypatch.setattr("builtins.input", Mock(side_effect=ValueError))
 
-        output = StringIO()
         console = ConsoleOutput(output)
         assert console.ask("test: ") is None
-        assert output.getvalue() == "test: "
+        assert getvalue(output) == "test: "
 
-    def test_askpass(self, monkeypatch: pytest.MonkeyPatch):
+    def test_askpass(self, monkeypatch: pytest.MonkeyPatch, output: TextIOWrapper):
         def getpass(prompt, stream):
             stream.write(prompt)
             return "hello"
 
         monkeypatch.setattr("streamlink_cli.console.getpass", getpass)
 
-        output = StringIO()
         console = ConsoleOutput(output)
         assert console.askpass("test: ") == "hello"
-        assert output.getvalue() == "test: "
+        assert getvalue(output) == "test: "
 
-    def test_askpass_no_tty(self):
-        output = StringIO()
+    def test_askpass_no_tty(self, output: TextIOWrapper):
         console = ConsoleOutput(output)
         assert console.askpass("test: ") is None
