Issue #8260: The read(), readline() and readlines() methods of

serhiy-storchaka · serhiy-storchaka · commit dbe0982bc515 · 2014-01-26T19:27:56.000+02:00
codecs.StreamReader returned incomplete data when were called after
readline() or read(size).  Based on patch by Amaury Forgeot d'Arc.
diff --git a/Lib/codecs.py b/Lib/codecs.py
@@ -475,22 +475,21 @@ def read(self, size=-1, chars=-1, firstline=False):
         # read until we get the required number of characters (if available)
         while True:
             # can the request be satisfied from the character buffer?
-            if chars < 0:
-                if size < 0:
-                    if self.charbuffer:
-                        break
-                elif len(self.charbuffer) >= size:
-                    break
-            else:
+            if chars >= 0:
                 if len(self.charbuffer) >= chars:
                     break
+            elif size >= 0:
+                if len(self.charbuffer) >= size:
+                    break
             # we need more data
             if size < 0:
                 newdata = self.stream.read()
             else:
                 newdata = self.stream.read(size)
             # decode bytes (those remaining from the last call included)
             data = self.bytebuffer + newdata
+            if not data:
+                break
             try:
                 newchars, decodedbytes = self.decode(data, self.errors)
             except UnicodeDecodeError as exc:
diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
@@ -175,6 +175,40 @@ def readalllines(input, keepends=True, size=None):
                         size*"a",
                     )
 
+    def test_mixed_readline_and_read(self):
+        lines = ["Humpty Dumpty sat on a wall,\n",
+                 "Humpty Dumpty had a great fall.\r\n",
+                 "All the king's horses and all the king's men\r",
+                 "Couldn't put Humpty together again."]
+        data = ''.join(lines)
+        def getreader():
+            stream = io.BytesIO(data.encode(self.encoding))
+            return codecs.getreader(self.encoding)(stream)
+
+        # Issue #8260: Test readline() followed by read()
+        f = getreader()
+        self.assertEqual(f.readline(), lines[0])
+        self.assertEqual(f.read(), ''.join(lines[1:]))
+        self.assertEqual(f.read(), '')
+
+        # Issue #16636: Test readline() followed by readlines()
+        f = getreader()
+        self.assertEqual(f.readline(), lines[0])
+        self.assertEqual(f.readlines(), lines[1:])
+        self.assertEqual(f.read(), '')
+
+        # Test read() followed by read()
+        f = getreader()
+        self.assertEqual(f.read(size=40, chars=5), data[:5])
+        self.assertEqual(f.read(), data[5:])
+        self.assertEqual(f.read(), '')
+
+        # Issue #12446: Test read() followed by readlines()
+        f = getreader()
+        self.assertEqual(f.read(size=40, chars=5), data[:5])
+        self.assertEqual(f.readlines(), [lines[0][5:]] + lines[1:])
+        self.assertEqual(f.read(), '')
+
     def test_bug1175396(self):
         s = [
             '<%!--===================================================\r\n',
@@ -2370,8 +2404,6 @@ def test_read(self):
 
     def test_readline(self):
         for encoding in bytes_transform_encodings:
-            if encoding in ['uu_codec', 'zlib_codec']:
-                continue
             with self.subTest(encoding=encoding):
                 sin = codecs.encode(b"\x80", encoding)
                 reader = codecs.getreader(encoding)(io.BytesIO(sin))
diff --git a/Misc/NEWS b/Misc/NEWS
@@ -36,6 +36,10 @@ Core and Builtins
 Library
 -------
 
+- Issue #8260: The read(), readline() and readlines() methods of
+  codecs.StreamReader returned incomplete data when were called after
+  readline() or read(size).  Based on patch by Amaury Forgeot d'Arc.
+
 - Issue #20105: the codec exception chaining now correctly sets the
   traceback of the original exception as its __traceback__ attribute.
 
