Merged revisions 77288 via svnmerge from

pitrou · pitrou · commit b1f8835b2134 · 2010-01-03T22:37:40.000Z
svn+ssh://pythondev@svn.python.org/python/trunk ........ r77288 | antoine.pitrou | 2010-01-03 23:29:56 +0100 (dim., 03 janv. 2010) | 5 lines Issue #7471: Improve the performance of GzipFile's buffering mechanism, and make it implement the `io.BufferedIOBase` ABC to allow for further speedups by wrapping it in an `io.BufferedReader`. Patch by Nir Aides. ........
diff --git a/Lib/gzip.py b/Lib/gzip.py
@@ -8,6 +8,7 @@
 import struct, sys, time, os
 import zlib
 import builtins
+import io
 
 __all__ = ["GzipFile","open"]
 
@@ -44,7 +45,7 @@ def open(filename, mode="rb", compresslevel=9):
     """
     return GzipFile(filename, mode, compresslevel)
 
-class GzipFile:
+class GzipFile(io.BufferedIOBase):
     """The GzipFile class simulates most of the methods of a file object with
     the exception of the readinto() and truncate() methods.
 
@@ -109,8 +110,12 @@ def __init__(self, filename=None, mode=None,
             self.mode = READ
             # Set flag indicating start of a new member
             self._new_member = True
+            # Buffer data read from gzip file. extrastart is offset in
+            # stream where buffer starts. extrasize is number of
+            # bytes remaining in buffer from current stream position.
             self.extrabuf = b""
             self.extrasize = 0
+            self.extrastart = 0
             self.name = filename
             # Starts small, scales exponentially
             self.min_readsize = 100
@@ -214,20 +219,26 @@ def _read_gzip_header(self):
         if flag & FHCRC:
             self.fileobj.read(2)     # Read & discard the 16-bit header CRC
 
-
     def write(self,data):
         if self.mode != WRITE:
             import errno
             raise IOError(errno.EBADF, "write() on read-only GzipFile object")
 
         if self.fileobj is None:
             raise ValueError("write() on closed GzipFile object")
+
+        # Convert data type if called by io.BufferedWriter.
+        if isinstance(data, memoryview):
+            data = data.tobytes()
+
         if len(data) > 0:
             self.size = self.size + len(data)
             self.crc = zlib.crc32(data, self.crc) & 0xffffffff
             self.fileobj.write( self.compress.compress(data) )
             self.offset += len(data)
 
+        return len(data)
+
     def read(self, size=-1):
         if self.mode != READ:
             import errno
@@ -253,15 +264,14 @@ def read(self, size=-1):
                 if size > self.extrasize:
                     size = self.extrasize
 
-        chunk = self.extrabuf[:size]
-        self.extrabuf = self.extrabuf[size:]
+        offset = self.offset - self.extrastart
+        chunk = self.extrabuf[offset: offset + size]
         self.extrasize = self.extrasize - size
 
         self.offset += size
         return chunk
 
     def _unread(self, buf):
-        self.extrabuf = buf + self.extrabuf
         self.extrasize = len(buf) + self.extrasize
         self.offset -= len(buf)
 
@@ -317,8 +327,10 @@ def _read(self, size=1024):
 
     def _add_read_data(self, data):
         self.crc = zlib.crc32(data, self.crc) & 0xffffffff
-        self.extrabuf = self.extrabuf + data
+        offset = self.offset - self.extrastart
+        self.extrabuf = self.extrabuf[offset:] + data
         self.extrasize = self.extrasize + len(data)
+        self.extrastart = self.offset
         self.size = self.size + len(data)
 
     def _read_eof(self):
@@ -336,6 +348,10 @@ def _read_eof(self):
         elif isize != (self.size & 0xffffffff):
             raise IOError("Incorrect length of data produced")
 
+    @property
+    def closed(self):
+        return self.fileobj is None
+
     def close(self):
         if self.fileobj is None:
             return
@@ -351,15 +367,6 @@ def close(self):
             self.myfileobj.close()
             self.myfileobj = None
 
-    def __del__(self):
-        try:
-            if (self.myfileobj is None and
-                self.fileobj is None):
-                return
-        except AttributeError:
-            return
-        self.close()
-
     def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH):
         if self.mode == WRITE:
             # Ensure the compressor's buffer is flushed
@@ -374,12 +381,6 @@ def fileno(self):
         """
         return self.fileobj.fileno()
 
-    def isatty(self):
-        return False
-
-    def tell(self):
-        return self.offset
-
     def rewind(self):
         '''Return the uncompressed stream file position indicator to the
         beginning of the file'''
@@ -389,8 +390,18 @@ def rewind(self):
         self._new_member = True
         self.extrabuf = b""
         self.extrasize = 0
+        self.extrastart = 0
         self.offset = 0
 
+    def readable(self):
+        return self.mode == READ
+
+    def writable(self):
+        return self.mode == WRITE
+
+    def seekable(self):
+        return True
+
     def seek(self, offset, whence=0):
         if whence:
             if whence == 1:
@@ -414,8 +425,18 @@ def seek(self, offset, whence=0):
                 self.read(1024)
             self.read(count % 1024)
 
+        return self.offset
+
     def readline(self, size=-1):
         if size < 0:
+            # Shortcut common case - newline found in buffer.
+            offset = self.offset - self.extrastart
+            i = self.extrabuf.find(b'\n', offset) + 1
+            if i > 0:
+                self.extrasize -= i - offset
+                self.offset += i - offset
+                return self.extrabuf[offset: i]
+
             size = sys.maxsize
             readsize = self.min_readsize
         else:
@@ -445,42 +466,6 @@ def readline(self, size=-1):
             self.min_readsize = min(readsize, self.min_readsize * 2, 512)
         return b''.join(bufs) # Return resulting line
 
-    def readlines(self, sizehint=0):
-        # Negative numbers result in reading all the lines
-        if sizehint <= 0:
-            sizehint = sys.maxsize
-        L = []
-        while sizehint > 0:
-            line = self.readline()
-            if line == b"":
-                break
-            L.append(line)
-            sizehint = sizehint - len(line)
-
-        return L
-
-    def writelines(self, L):
-        for line in L:
-            self.write(line)
-
-    def __iter__(self):
-        return self
-
-    def __next__(self):
-        line = self.readline()
-        if line:
-            return line
-        else:
-            raise StopIteration
-
-    def __enter__(self):
-        if self.fileobj is None:
-            raise ValueError("I/O operation on closed GzipFile object")
-        return self
-
-    def __exit__(self, *args):
-        self.close()
-
 
 def _test():
     # Act like gzip; with -d, act like gunzip.
diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py
@@ -5,6 +5,7 @@
 import unittest
 from test import support
 import os
+import io
 import struct
 gzip = support.import_module('gzip')
 
@@ -80,6 +81,16 @@ def test_many_append(self):
         zgfile.close()
         self.assertEquals(contents, b'a'*201)
 
+    def test_buffered_reader(self):
+        # Issue #7471: a GzipFile can be wrapped in a BufferedReader for
+        # performance.
+        self.test_write()
+
+        f = gzip.GzipFile(self.filename, 'rb')
+        with io.BufferedReader(f) as r:
+            lines = [line for line in r]
+
+        self.assertEqual(lines, 50 * data1.splitlines(True))
 
     def test_readline(self):
         self.test_write()
diff --git a/Misc/NEWS b/Misc/NEWS
@@ -191,7 +191,11 @@ C-API
 Library
 -------
 
-_ Issue #3972: http.client.HTTPConnection now accepts an optional source_address
+- Issue #7471: Improve the performance of GzipFile's buffering mechanism,
+  and make it implement the `io.BufferedIOBase` ABC to allow for further
+  speedups by wrapping it in an `io.BufferedReader`.  Patch by Nir Aides.
+
+- Issue #3972: http.client.HTTPConnection now accepts an optional source_address
   parameter to allow specifying where your connections come from.
 
 - socket.create_connection now accepts an optional source_address parameter.
