Issue #13333: The UTF-7 decoder now accepts lone surrogates

pitrou · pitrou · commit 78edf7576e4b · 2011-11-15T01:44:16.000+01:00
(the encoder already accepts them).
diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py
@@ -1108,10 +1108,18 @@ def test_codecs_utf7(self):
         for (x, y) in utfTests:
             self.assertEqual(x.encode('utf-7'), y)
 
-        # Unpaired surrogates not supported
-        self.assertRaises(UnicodeError, str, b'+3ADYAA-', 'utf-7')
-
-        self.assertEqual(str(b'+3ADYAA-', 'utf-7', 'replace'), '\ufffd\ufffd')
+        # Unpaired surrogates are passed through
+        self.assertEqual('\uD801'.encode('utf-7'), b'+2AE-')
+        self.assertEqual('\uD801x'.encode('utf-7'), b'+2AE-x')
+        self.assertEqual('\uDC01'.encode('utf-7'), b'+3AE-')
+        self.assertEqual('\uDC01x'.encode('utf-7'), b'+3AE-x')
+        self.assertEqual(b'+2AE-'.decode('utf-7'), '\uD801')
+        self.assertEqual(b'+2AE-x'.decode('utf-7'), '\uD801x')
+        self.assertEqual(b'+3AE-'.decode('utf-7'), '\uDC01')
+        self.assertEqual(b'+3AE-x'.decode('utf-7'), '\uDC01x')
+
+        self.assertEqual('\uD801\U000abcde'.encode('utf-7'), b'+2AHab9ze-')
+        self.assertEqual(b'+2AHab9ze-'.decode('utf-7'), '\uD801\U000abcde')
 
         # Issue #2242: crash on some Windows/MSVC versions
         self.assertEqual(b'+\xc1'.decode('utf-7'), '\xc1')
diff --git a/Misc/NEWS b/Misc/NEWS
@@ -10,6 +10,9 @@ What's New in Python 3.3 Alpha 1?
 Core and Builtins
 -----------------
 
+- Issue #13333: The UTF-7 decoder now accepts lone surrogates (the encoder
+  already accepts them).
+
 - Issue #13389: Full garbage collection passes now clear the freelists for
   list and dict objects.  They already cleared other freelists in the
   interpreter.
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
@@ -3884,21 +3884,18 @@ PyUnicode_DecodeUTF7Stateful(const char *s,
                             if (unicode_putchar(&unicode, &outpos, ch2) < 0)
                                 goto onError;
                             surrogate = 0;
+                            continue;
                         }
                         else {
+                            if (unicode_putchar(&unicode, &outpos, surrogate) < 0)
+                                goto onError;
                             surrogate = 0;
-                            errmsg = "second surrogate missing";
-                            goto utf7Error;
                         }
                     }
-                    else if (outCh >= 0xD800 && outCh <= 0xDBFF) {
+                    if (outCh >= 0xD800 && outCh <= 0xDBFF) {
                         /* first surrogate */
                         surrogate = outCh;
                     }
-                    else if (outCh >= 0xDC00 && outCh <= 0xDFFF) {
-                        errmsg = "unexpected second surrogate";
-                        goto utf7Error;
-                    }
                     else {
                         if (unicode_putchar(&unicode, &outpos, outCh) < 0)
                             goto onError;
@@ -3909,8 +3906,9 @@ PyUnicode_DecodeUTF7Stateful(const char *s,
                 inShift = 0;
                 s++;
                 if (surrogate) {
-                    errmsg = "second surrogate missing at end of shift sequence";
-                    goto utf7Error;
+                    if (unicode_putchar(&unicode, &outpos, surrogate) < 0)
+                        goto onError;
+                    surrogate = 0;
                 }
                 if (base64bits > 0) { /* left-over bits */
                     if (base64bits >= 6) {
