Minimal fix for the complaints about pickling Unicode objects. (SF

gvanrossum · gvanrossum · commit fb10c3f6645c · 2000-12-19T02:08:38.000Z
bugs #126161 and 123634). The solution doesn't use the unicode-escape encoding; that has other problems (it seems not 100% reversible). Rather, it transforms the input Unicode object slightly before encoding it using raw-unicode-escape, so that the decoding will reconstruct the original string: backslash and newline characters are translated into their \uXXXX counterparts. This is backwards incompatible for strings containing backslashes, but for some of those strings, the pickling was already broken. Note that SF bug #123634 complains specifically that cPickle fails to unpickle the pickle for u'' (the empty Unicode string) correctly. This was an off-by-one error in load_unicode(). XXX Ugliness: in order to do the modified raw-unicode-escape, I've cut-and-pasted a copy of PyUnicode_EncodeRawUnicodeEscape() into this file that also encodes '\\' and '\n'. It might be nice to migrate this into the Unicode implementation and give this encoding a new name ('half-raw-unicode-escape'? 'pickle-unicode-escape'?); that would help pickle.py too. But right now I can't be bothered with the necessary infrastructural changes.
diff --git a/Modules/cPickle.c b/Modules/cPickle.c
@@ -1149,6 +1149,51 @@ save_string(Picklerobject *self, PyObject *args, int doput) {
 }
 
 
+/* A copy of PyUnicode_EncodeRawUnicodeEscape() that also translates
+   backslash and newline characters to \uXXXX escapes. */
+static PyObject *
+modified_EncodeRawUnicodeEscape(const Py_UNICODE *s, int size)
+{
+    PyObject *repr;
+    char *p;
+    char *q;
+
+    static const char *hexdigit = "0123456789ABCDEF";
+
+    repr = PyString_FromStringAndSize(NULL, 6 * size);
+    if (repr == NULL)
+        return NULL;
+    if (size == 0)
+	return repr;
+
+    p = q = PyString_AS_STRING(repr);
+    while (size-- > 0) {
+        Py_UNICODE ch = *s++;
+	/* Map 16-bit characters to '\uxxxx' */
+	if (ch >= 256 || ch == '\\' || ch == '\n') {
+            *p++ = '\\';
+            *p++ = 'u';
+            *p++ = hexdigit[(ch >> 12) & 0xf];
+            *p++ = hexdigit[(ch >> 8) & 0xf];
+            *p++ = hexdigit[(ch >> 4) & 0xf];
+            *p++ = hexdigit[ch & 15];
+        }
+	/* Copy everything else as-is */
+	else
+            *p++ = (char) ch;
+    }
+    *p = '\0';
+    if (_PyString_Resize(&repr, p - q))
+	goto onError;
+
+    return repr;
+
+ onError:
+    Py_DECREF(repr);
+    return NULL;
+}
+
+
 static int
 save_unicode(Picklerobject *self, PyObject *args, int doput) {
     int size, len;
@@ -1161,7 +1206,8 @@ save_unicode(Picklerobject *self, PyObject *args, int doput) {
         char *repr_str;
         static char string = UNICODE;
 
-        UNLESS (repr = PyUnicode_AsRawUnicodeEscapeString(args))
+        UNLESS(repr = modified_EncodeRawUnicodeEscape(
+		PyUnicode_AS_UNICODE(args), PyUnicode_GET_SIZE(args)))
             return -1;
 
         if ((len = PyString_Size(repr)) < 0)
@@ -2745,7 +2791,7 @@ load_unicode(Unpicklerobject *self) {
     char *s;
 
     if ((len = (*self->readline_func)(self, &s)) < 0) return -1;
-    if (len < 2) return bad_readline();
+    if (len < 1) return bad_readline();
 
     UNLESS (str = PyUnicode_DecodeRawUnicodeEscape(s, len - 1, NULL))
         goto finally;
