Issue #19886: Use better estimated memory requirements for bigmem tests.

serhiy-storchaka · serhiy-storchaka · commit 4847e4e1f47b · 2014-01-10T13:37:54.000+02:00
Incorrect requirements can cause memory swapping.
diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py
@@ -19,8 +19,6 @@
 # kind of outer loop.
 protocols = range(pickle.HIGHEST_PROTOCOL + 1)
 
-ascii_char_size = 1
-
 
 # Return True if opcode code appears in the pickle, else False.
 def opcode_in_pickle(code, pickle):
@@ -1331,7 +1329,7 @@ class BigmemPickleTests(unittest.TestCase):
 
     # Binary protocols can serialize longs of up to 2GB-1
 
-    @bigmemtest(size=_2G, memuse=1 + 1, dry_run=False)
+    @bigmemtest(size=_2G, memuse=3.6, dry_run=False)
     def test_huge_long_32b(self, size):
         data = 1 << (8 * size)
         try:
@@ -1347,7 +1345,7 @@ def test_huge_long_32b(self, size):
     # (older protocols don't have a dedicated opcode for bytes and are
     # too inefficient)
 
-    @bigmemtest(size=_2G, memuse=1 + 1, dry_run=False)
+    @bigmemtest(size=_2G, memuse=2.5, dry_run=False)
     def test_huge_bytes_32b(self, size):
         data = b"abcd" * (size // 4)
         try:
@@ -1363,7 +1361,7 @@ def test_huge_bytes_32b(self, size):
         finally:
             data = None
 
-    @bigmemtest(size=_4G, memuse=1 + 1, dry_run=False)
+    @bigmemtest(size=_4G, memuse=2.5, dry_run=False)
     def test_huge_bytes_64b(self, size):
         data = b"a" * size
         try:
@@ -1378,7 +1376,7 @@ def test_huge_bytes_64b(self, size):
     # All protocols use 1-byte per printable ASCII character; we add another
     # byte because the encoded form has to be copied into the internal buffer.
 
-    @bigmemtest(size=_2G, memuse=2 + ascii_char_size, dry_run=False)
+    @bigmemtest(size=_2G, memuse=8, dry_run=False)
     def test_huge_str_32b(self, size):
         data = "abcd" * (size // 4)
         try:
@@ -1395,7 +1393,7 @@ def test_huge_str_32b(self, size):
     # BINUNICODE (protocols 1, 2 and 3) cannot carry more than
     # 2**32 - 1 bytes of utf-8 encoded unicode.
 
-    @bigmemtest(size=_4G, memuse=1 + ascii_char_size, dry_run=False)
+    @bigmemtest(size=_4G, memuse=8, dry_run=False)
     def test_huge_str_64b(self, size):
         data = "a" * size
         try:
diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py
@@ -679,7 +679,7 @@ def testEOFError(self):
         self.assertRaises(EOFError, bz2d.decompress, b"anything")
         self.assertRaises(EOFError, bz2d.decompress, b"")
 
-    @bigmemtest(size=_4G + 100, memuse=3)
+    @bigmemtest(size=_4G + 100, memuse=3.3)
     def testDecompress4G(self, size):
         # "Test BZ2Decompressor.decompress() with >4GiB input"
         blocksize = 10 * 1024 * 1024
diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py
@@ -235,21 +235,15 @@ def test_case_md5_2(self):
                    b'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789',
                    'd174ab98d277d9f5a5611c2c9f419d9f')
 
-    @bigmemtest(size=_4G + 5, memuse=1)
+    @unittest.skipIf(sys.maxsize < _4G + 5, 'test cannot run on 32-bit systems')
+    @bigmemtest(size=_4G + 5, memuse=1, dry_run=False)
     def test_case_md5_huge(self, size):
-        if size == _4G + 5:
-            try:
-                self.check('md5', b'A'*size, 'c9af2dff37468ce5dfee8f2cfc0a9c6d')
-            except OverflowError:
-                pass # 32-bit arch
+        self.check('md5', b'A'*size, 'c9af2dff37468ce5dfee8f2cfc0a9c6d')
 
-    @bigmemtest(size=_4G - 1, memuse=1)
+    @unittest.skipIf(sys.maxsize < _4G - 1, 'test cannot run on 32-bit systems')
+    @bigmemtest(size=_4G - 1, memuse=1, dry_run=False)
     def test_case_md5_uintmax(self, size):
-        if size == _4G - 1:
-            try:
-                self.check('md5', b'A'*size, '28138d306ff1b8281f1a9067e1a1a2b3')
-            except OverflowError:
-                pass # 32-bit arch
+        self.check('md5', b'A'*size, '28138d306ff1b8281f1a9067e1a1a2b3')
 
     # use the three examples from Federal Information Processing Standards
     # Publication 180-1, Secure Hash Standard,  1995 April 17
diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py
@@ -303,19 +303,19 @@ class LargeValuesTestCase(unittest.TestCase):
     def check_unmarshallable(self, data):
         self.assertRaises(ValueError, marshal.dump, data, NullWriter())
 
-    @support.bigmemtest(size=LARGE_SIZE, memuse=1, dry_run=False)
+    @support.bigmemtest(size=LARGE_SIZE, memuse=2, dry_run=False)
     def test_bytes(self, size):
         self.check_unmarshallable(b'x' * size)
 
-    @support.bigmemtest(size=LARGE_SIZE, memuse=1, dry_run=False)
+    @support.bigmemtest(size=LARGE_SIZE, memuse=2, dry_run=False)
     def test_str(self, size):
         self.check_unmarshallable('x' * size)
 
-    @support.bigmemtest(size=LARGE_SIZE, memuse=pointer_size, dry_run=False)
+    @support.bigmemtest(size=LARGE_SIZE, memuse=pointer_size + 1, dry_run=False)
     def test_tuple(self, size):
         self.check_unmarshallable((None,) * size)
 
-    @support.bigmemtest(size=LARGE_SIZE, memuse=pointer_size, dry_run=False)
+    @support.bigmemtest(size=LARGE_SIZE, memuse=pointer_size + 1, dry_run=False)
     def test_list(self, size):
         self.check_unmarshallable([None] * size)
 
@@ -331,7 +331,7 @@ def test_set(self, size):
     def test_frozenset(self, size):
         self.check_unmarshallable(frozenset(range(size)))
 
-    @support.bigmemtest(size=LARGE_SIZE, memuse=1, dry_run=False)
+    @support.bigmemtest(size=LARGE_SIZE, memuse=2, dry_run=False)
     def test_bytearray(self, size):
         self.check_unmarshallable(bytearray(size))
 
diff --git a/Lib/test/test_xml_etree_c.py b/Lib/test/test_xml_etree_c.py
@@ -12,10 +12,8 @@
 
 class MiscTests(unittest.TestCase):
     # Issue #8651.
-    @support.bigmemtest(size=support._2G + 100, memuse=1)
+    @support.bigmemtest(size=support._2G + 100, memuse=1, dry_run=False)
     def test_length_overflow(self, size):
-        if size < support._2G + 100:
-            self.skipTest("not enough free memory, need at least 2 GB")
         data = b'x' * size
         parser = cET.XMLParser()
         try:
diff --git a/Misc/NEWS b/Misc/NEWS
@@ -234,6 +234,8 @@ IDLE
 Tests
 -----
 
+- Issue #19886: Use better estimated memory requirements for bigmem tests.
+
 - Issue #20055: Fix test_shutil under Windows with symlink privileges held.
   Patch by Vajrasky Kok.
 
