bpo-19865: ctypes.create_unicode_buffer() supports non-BMP strings on Windows (GH-14081)

ZackerySpytz · vstinner · commit 9765efcb39fc · 2019-06-14T17:53:59.000+02:00
diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py
@@ -274,7 +274,15 @@ def create_unicode_buffer(init, size=None):
     """
     if isinstance(init, str):
         if size is None:
-            size = len(init)+1
+            if sizeof(c_wchar) == 2:
+                # UTF-16 requires a surrogate pair (2 wchar_t) for non-BMP
+                # characters (outside [U+0000; U+FFFF] range). +1 for trailing
+                # NUL character.
+                size = sum(2 if ord(c) > 0xFFFF else 1 for c in init) + 1
+            else:
+                # 32-bit wchar_t (1 wchar_t per Unicode character). +1 for
+                # trailing NUL character.
+                size = len(init) + 1
         buftype = c_wchar * size
         buf = buftype()
         buf.value = init
diff --git a/Lib/ctypes/test/test_buffers.py b/Lib/ctypes/test/test_buffers.py
@@ -60,5 +60,14 @@ def test_unicode_conversion(self):
         self.assertEqual(b[::2], "ac")
         self.assertEqual(b[::5], "a")
 
+    @need_symbol('c_wchar')
+    def test_create_unicode_buffer_non_bmp(self):
+        expected = 5 if sizeof(c_wchar) == 2 else 3
+        for s in '\U00010000\U00100000', '\U00010000\U0010ffff':
+            b = create_unicode_buffer(s)
+            self.assertEqual(len(b), expected)
+            self.assertEqual(b[-1], '\0')
+
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/Misc/NEWS.d/next/Library/2019-06-14-08-30-16.bpo-19865.FRGH4I.rst b/Misc/NEWS.d/next/Library/2019-06-14-08-30-16.bpo-19865.FRGH4I.rst
@@ -0,0 +1,2 @@
+:func:`ctypes.create_unicode_buffer()` now also supports non-BMP characters
+on platforms with 16-bit :c:type:`wchar_t` (for example, Windows and AIX).

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	+:func:`ctypes.create_unicode_buffer()` now also supports non-BMP characters
	`2`	+on platforms with 16-bit :c:type:`wchar_t` (for example, Windows and AIX).