Merge #13159: Replace FileIO's quadratic-time buffer growth algorithm with a linear-time one.

nvawda · nvawda · commit 72d6a13413d3 · 2011-10-13T13:38:14.000+02:00
Also fix the bz2 module, which suffered from the same problem.
diff --git a/Misc/NEWS b/Misc/NEWS
@@ -1342,6 +1342,9 @@ Tools/Demos
 Extension Modules
 -----------------
 
+- Issue #13159: FileIO and BZ2Compressor/BZ2Decompressor now use a linear-time
+  buffer growth strategy instead of a quadratic-time one.
+
 - Issue #10141: socket: Add SocketCAN (PF_CAN) support. Initial patch by
   Matthias Fuchs, updated by Tiago Gonçalves.
 
diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c
@@ -116,22 +116,14 @@ catch_bz2_error(int bzerror)
 #define SMALLCHUNK BUFSIZ
 #endif
 
-#if SIZEOF_INT < 4
-#define BIGCHUNK  (512 * 32)
-#else
-#define BIGCHUNK  (512 * 1024)
-#endif
-
 static int
 grow_buffer(PyObject **buf)
 {
+    /* Expand the buffer by an amount proportional to the current size,
+       giving us amortized linear-time behavior. Use a less-than-double
+       growth factor to avoid excessive allocation. */
     size_t size = PyBytes_GET_SIZE(*buf);
-    if (size <= SMALLCHUNK)
-        return _PyBytes_Resize(buf, size + SMALLCHUNK);
-    else if (size <= BIGCHUNK)
-        return _PyBytes_Resize(buf, size * 2);
-    else
-        return _PyBytes_Resize(buf, size + BIGCHUNK);
+    return _PyBytes_Resize(buf, size + (size >> 3) + 6);
 }
 
 
diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c
@@ -43,12 +43,6 @@
 #define SMALLCHUNK BUFSIZ
 #endif
 
-#if SIZEOF_INT < 4
-#define BIGCHUNK  (512 * 32)
-#else
-#define BIGCHUNK  (512 * 1024)
-#endif
-
 typedef struct {
     PyObject_HEAD
     int fd;
@@ -572,15 +566,10 @@ new_buffersize(fileio *self, size_t currentsize
         }
     }
 #endif
-    if (currentsize > SMALLCHUNK) {
-        /* Keep doubling until we reach BIGCHUNK;
-           then keep adding BIGCHUNK. */
-        if (currentsize <= BIGCHUNK)
-            return currentsize + currentsize;
-        else
-            return currentsize + BIGCHUNK;
-    }
-    return currentsize + SMALLCHUNK;
+    /* Expand the buffer by an amount proportional to the current size,
+       giving us amortized linear-time behavior. Use a less-than-double
+       growth factor to avoid excessive allocation. */
+    return currentsize + (currentsize >> 3) + 6;
 }
 
 static PyObject *
