Issue #12591: Allow io.TextIOWrapper to work with raw IO objects (without

pitrou · pitrou · commit d42c1d09e99d · 2011-07-23T21:50:21.000+02:00
a read1() method), and add a *write_through* parameter to
mandate unbuffered writes.
diff --git a/Lib/_pyio.py b/Lib/_pyio.py
@@ -1478,7 +1478,7 @@ class TextIOWrapper(TextIOBase):
     _CHUNK_SIZE = 2048
 
     def __init__(self, buffer, encoding=None, errors=None, newline=None,
-                 line_buffering=False):
+                 line_buffering=False, write_through=False):
         if newline is not None and not isinstance(newline, str):
             raise TypeError("illegal newline type: %r" % (type(newline),))
         if newline not in (None, "", "\n", "\r", "\r\n"):
@@ -1521,6 +1521,7 @@ def __init__(self, buffer, encoding=None, errors=None, newline=None,
         self._decoded_chars_used = 0  # offset into _decoded_chars for read()
         self._snapshot = None  # info for reconstructing decoder state
         self._seekable = self._telling = self.buffer.seekable()
+        self._has_read1 = hasattr(self.buffer, 'read1')
         self._b2cratio = 0.0
 
         if self._seekable and self.writable():
@@ -1687,7 +1688,10 @@ def _read_chunk(self):
             # len(dec_buffer) bytes ago with decoder state (b'', dec_flags).
 
         # Read a chunk, decode it, and put the result in self._decoded_chars.
-        input_chunk = self.buffer.read1(self._CHUNK_SIZE)
+        if self._has_read1:
+            input_chunk = self.buffer.read1(self._CHUNK_SIZE)
+        else:
+            input_chunk = self.buffer.read(self._CHUNK_SIZE)
         eof = not input_chunk
         decoded_chars = self._decoder.decode(input_chunk, eof)
         self._set_decoded_chars(decoded_chars)
diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py
@@ -2314,6 +2314,27 @@ def test_readonly_attributes(self):
         with self.assertRaises(AttributeError):
             txt.buffer = buf
 
+    def test_rawio(self):
+        # Issue #12591: TextIOWrapper must work with raw I/O objects, so
+        # that subprocess.Popen() can have the required unbuffered
+        # semantics with universal_newlines=True.
+        raw = self.MockRawIO([b'abc', b'def', b'ghi\njkl\nopq\n'])
+        txt = self.TextIOWrapper(raw, encoding='ascii', newline='\n')
+        # Reads
+        self.assertEqual(txt.read(4), 'abcd')
+        self.assertEqual(txt.readline(), 'efghi\n')
+        self.assertEqual(list(txt), ['jkl\n', 'opq\n'])
+
+    def test_rawio_write_through(self):
+        # Issue #12591: with write_through=True, writes don't need a flush
+        raw = self.MockRawIO([b'abc', b'def', b'ghi\njkl\nopq\n'])
+        txt = self.TextIOWrapper(raw, encoding='ascii', newline='\n',
+                                 write_through=True)
+        txt.write('1')
+        txt.write('23\n4')
+        txt.write('5')
+        self.assertEqual(b''.join(raw._write_stack), b'123\n45')
+
 class CTextIOWrapperTest(TextIOWrapperTest):
 
     def test_initialization(self):
diff --git a/Misc/NEWS b/Misc/NEWS
@@ -237,6 +237,10 @@ Core and Builtins
 Library
 -------
 
+- Issue #12591: Allow io.TextIOWrapper to work with raw IO objects (without
+  a read1() method), and add a *write_through* parameter to mandate
+  unbuffered writes.
+
 - Issue #10883: Fix socket leaks in urllib.request when using FTP.
 
 - Issue #12592: Make Python build on OpenBSD 5 (and future major releases).
diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c
@@ -653,10 +653,12 @@ typedef struct
     PyObject *errors;
     const char *writenl; /* utf-8 encoded, NULL stands for \n */
     char line_buffering;
+    char write_through;
     char readuniversal;
     char readtranslate;
     char writetranslate;
     char seekable;
+    char has_read1;
     char telling;
     char deallocating;
     /* Specialized encoding func (see below) */
@@ -813,23 +815,23 @@ static int
 textiowrapper_init(textio *self, PyObject *args, PyObject *kwds)
 {
     char *kwlist[] = {"buffer", "encoding", "errors",
-                      "newline", "line_buffering",
+                      "newline", "line_buffering", "write_through",
                       NULL};
     PyObject *buffer, *raw;
     char *encoding = NULL;
     char *errors = NULL;
     char *newline = NULL;
-    int line_buffering = 0;
+    int line_buffering = 0, write_through = 0;
     _PyIO_State *state = IO_STATE;
 
     PyObject *res;
     int r;
 
     self->ok = 0;
     self->detached = 0;
-    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|zzzi:fileio",
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|zzzii:fileio",
                                      kwlist, &buffer, &encoding, &errors,
-                                     &newline, &line_buffering))
+                                     &newline, &line_buffering, &write_through))
         return -1;
 
     if (newline && newline[0] != '\0'
@@ -935,6 +937,7 @@ textiowrapper_init(textio *self, PyObject *args, PyObject *kwds)
     self->chunk_size = 8192;
     self->readuniversal = (newline == NULL || newline[0] == '\0');
     self->line_buffering = line_buffering;
+    self->write_through = write_through;
     self->readtranslate = (newline == NULL);
     if (newline) {
         self->readnl = PyUnicode_FromString(newline);
@@ -1044,6 +1047,8 @@ textiowrapper_init(textio *self, PyObject *args, PyObject *kwds)
     self->seekable = self->telling = PyObject_IsTrue(res);
     Py_DECREF(res);
 
+    self->has_read1 = PyObject_HasAttrString(buffer, "read1");
+
     self->encoding_start_of_stream = 0;
     if (self->seekable && self->encoder) {
         PyObject *cookieObj;
@@ -1287,7 +1292,9 @@ textiowrapper_write(textio *self, PyObject *args)
         text = newtext;
     }
 
-    if (self->line_buffering &&
+    if (self->write_through)
+        needflush = 1;
+    else if (self->line_buffering &&
         (haslf ||
          findchar(PyUnicode_AS_UNICODE(text),
                   PyUnicode_GET_SIZE(text), '\r')))
@@ -1435,7 +1442,8 @@ textiowrapper_read_chunk(textio *self)
     if (chunk_size == NULL)
         goto fail;
     input_chunk = PyObject_CallMethodObjArgs(self->buffer,
-        _PyIO_str_read1, chunk_size, NULL);
+        (self->has_read1 ? _PyIO_str_read1: _PyIO_str_read),
+        chunk_size, NULL);
     Py_DECREF(chunk_size);
     if (input_chunk == NULL)
         goto fail;
