From 39117316d31722103f871bf99c4569d34c7a836f Mon Sep 17 00:00:00 2001
From: Ammar Askar <ammar@ammaraskar.com>
Date: Fri, 24 Apr 2020 20:43:45 -0700
Subject: [PATCH 1/3] bpo-34990: Treat the pyc header's mtime in compileall as
 an unsigned int
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Stéphane Wirtel <stephane@wirtel.be>
---
 Lib/compileall.py                                |  4 ++--
 Lib/test/test_compileall.py                      | 16 ++++++++++++++--
 Lib/test/test_zipimport.py                       | 16 +++++++++++++++-
 .../2020-04-24-20-39-38.bpo-34990.3SmL9M.rst     |  2 ++
 4 files changed, 33 insertions(+), 5 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Library/2020-04-24-20-39-38.bpo-34990.3SmL9M.rst

diff --git a/Lib/compileall.py b/Lib/compileall.py
index 1831ad749f2b17..966bc4b2b55ea4 100644
--- a/Lib/compileall.py
+++ b/Lib/compileall.py
@@ -206,8 +206,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
             if not force:
                 try:
                     mtime = int(os.stat(fullname).st_mtime)
-                    expect = struct.pack('<4sll', importlib.util.MAGIC_NUMBER,
-                                         0, mtime)
+                    expect = struct.pack('<4sLL', importlib.util.MAGIC_NUMBER,
+                                         0, mtime & 0xFFFF_FFFF)
                     for cfile in opt_cfiles.values():
                         with open(cfile, 'rb') as chandle:
                             actual = chandle.read(12)
diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py
index cb59fd71b38254..60a634117f266a 100644
--- a/Lib/test/test_compileall.py
+++ b/Lib/test/test_compileall.py
@@ -55,9 +55,21 @@ def timestamp_metadata(self):
         with open(self.bc_path, 'rb') as file:
             data = file.read(12)
         mtime = int(os.stat(self.source_path).st_mtime)
-        compare = struct.pack('<4sll', importlib.util.MAGIC_NUMBER, 0, mtime)
+        compare = struct.pack('<4sLL', importlib.util.MAGIC_NUMBER, 0,
+                              mtime & 0xFFFF_FFFF)
         return data, compare
 
+    def test_year_2038_mtime_compilation(self):
+        # Test to make sure we can handle mtimes larger than what a 32-bit
+        # signed number can hold as part of bpo-34990
+        with open(self.source_path, 'r') as f:
+            os.utime(f.name, (2**32 - 1, 2**32 - 1))
+        self.assertTrue(compileall.compile_file(self.source_path))
+
+        with open(self.source_path, 'r') as f:
+            os.utime(f.name, (2**35, 2**35))
+        self.assertTrue(compileall.compile_file(self.source_path))
+
     def recreation_check(self, metadata):
         """Check that compileall recreates bytecode when the new metadata is
         used."""
@@ -76,7 +88,7 @@ def recreation_check(self, metadata):
 
     def test_mtime(self):
         # Test a change in mtime leads to a new .pyc.
-        self.recreation_check(struct.pack('<4sll', importlib.util.MAGIC_NUMBER,
+        self.recreation_check(struct.pack('<4sLL', importlib.util.MAGIC_NUMBER,
                                           0, 1))
 
     def test_magic_number(self):
diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py
index 2af8689c1d2cb2..8195d322405afe 100644
--- a/Lib/test/test_zipimport.py
+++ b/Lib/test/test_zipimport.py
@@ -41,7 +41,8 @@ def make_pyc(co, mtime, size):
         else:
             mtime = int(-0x100000000 + int(mtime))
     pyc = (importlib.util.MAGIC_NUMBER +
-        struct.pack("<iii", 0, int(mtime), size & 0xFFFFFFFF) + data)
+        struct.pack("<iLL", 0,
+                    int(mtime) & 0xFFFF_FFFF, size & 0xFFFF_FFFF) + data)
     return pyc
 
 def module_path_to_dotted_name(path):
@@ -253,6 +254,19 @@ def testBadMTime(self):
                  TESTMOD + pyc_ext: (NOW, badtime_pyc)}
         self.doTest(".py", files, TESTMOD)
 
+    def test2038MTime(self):
+        # Make sure we can handle mtimes larger than what a 32-bit signed number
+        # can hold.
+        twenty_thirty_eight_pyc = make_pyc(test_co, 2**32 - 1, len(test_src))
+        files = {TESTMOD + ".py": (NOW, test_src),
+                 TESTMOD + pyc_ext: (NOW, twenty_thirty_eight_pyc)}
+        self.doTest(".py", files, TESTMOD)
+
+        larger_than_32_bit = make_pyc(test_co, 2**35, len(test_src))
+        files = {TESTMOD + ".py": (NOW, test_src),
+                 TESTMOD + pyc_ext: (NOW, twenty_thirty_eight_pyc)}
+        self.doTest(".py", files, TESTMOD)
+
     def testPackage(self):
         packdir = TESTPACK + os.sep
         files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc),
diff --git a/Misc/NEWS.d/next/Library/2020-04-24-20-39-38.bpo-34990.3SmL9M.rst b/Misc/NEWS.d/next/Library/2020-04-24-20-39-38.bpo-34990.3SmL9M.rst
new file mode 100644
index 00000000000000..6a1f2ea88b08cf
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2020-04-24-20-39-38.bpo-34990.3SmL9M.rst
@@ -0,0 +1,2 @@
+Fixed a Y2k38 bug in the compileall module where it would fail to compile
+files created after 2038.

From 1071bf522107447caedc4ecd23f2613f7cab65e2 Mon Sep 17 00:00:00 2001
From: Ammar Askar <ammar@ammaraskar.com>
Date: Thu, 30 Apr 2020 00:43:43 -0700
Subject: [PATCH 2/3] Address Serhiy's comments

---
 Lib/test/test_compileall.py | 12 ++++++++----
 Lib/test/test_zipimport.py  | 11 -----------
 2 files changed, 8 insertions(+), 15 deletions(-)

diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py
index 60a634117f266a..2c09b00f2d06a3 100644
--- a/Lib/test/test_compileall.py
+++ b/Lib/test/test_compileall.py
@@ -62,12 +62,16 @@ def timestamp_metadata(self):
     def test_year_2038_mtime_compilation(self):
         # Test to make sure we can handle mtimes larger than what a 32-bit
         # signed number can hold as part of bpo-34990
-        with open(self.source_path, 'r') as f:
-            os.utime(f.name, (2**32 - 1, 2**32 - 1))
+        os.utime(self.source_path, (2**32 - 1, 2**32 - 1))
         self.assertTrue(compileall.compile_file(self.source_path))
 
-        with open(self.source_path, 'r') as f:
-            os.utime(f.name, (2**35, 2**35))
+    def test_larger_than_32_bit_times(self):
+        # This is similar to the test above but we skip it if the OS doesn't
+        # support modification times larger than 32-bits.
+        try:
+            os.utime(self.source_path, (2**35, 2**35))
+        except (OverflowError, OSError):
+            self.skipTest("filesystem doesn't support large timestamps")
         self.assertTrue(compileall.compile_file(self.source_path))
 
     def recreation_check(self, metadata):
diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py
index 8195d322405afe..a08c96698ab791 100644
--- a/Lib/test/test_zipimport.py
+++ b/Lib/test/test_zipimport.py
@@ -34,12 +34,6 @@ def get_file():
 
 def make_pyc(co, mtime, size):
     data = marshal.dumps(co)
-    if type(mtime) is type(0.0):
-        # Mac mtimes need a bit of special casing
-        if mtime < 0x7fffffff:
-            mtime = int(mtime)
-        else:
-            mtime = int(-0x100000000 + int(mtime))
     pyc = (importlib.util.MAGIC_NUMBER +
         struct.pack("<iLL", 0,
                     int(mtime) & 0xFFFF_FFFF, size & 0xFFFF_FFFF) + data)
@@ -262,11 +256,6 @@ def test2038MTime(self):
                  TESTMOD + pyc_ext: (NOW, twenty_thirty_eight_pyc)}
         self.doTest(".py", files, TESTMOD)
 
-        larger_than_32_bit = make_pyc(test_co, 2**35, len(test_src))
-        files = {TESTMOD + ".py": (NOW, test_src),
-                 TESTMOD + pyc_ext: (NOW, twenty_thirty_eight_pyc)}
-        self.doTest(".py", files, TESTMOD)
-
     def testPackage(self):
         packdir = TESTPACK + os.sep
         files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc),

From 91b54fd725897f5f136ee60fd778289a7ea66c8a Mon Sep 17 00:00:00 2001
From: Ammar Askar <ammar@ammaraskar.com>
Date: Thu, 30 Apr 2020 14:44:04 -0700
Subject: [PATCH 3/3] Address Victor's comments

---
 Lib/test/test_compileall.py                                  | 5 ++++-
 .../next/Library/2020-04-24-20-39-38.bpo-34990.3SmL9M.rst    | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py
index 2c09b00f2d06a3..7ca056fd0922af 100644
--- a/Lib/test/test_compileall.py
+++ b/Lib/test/test_compileall.py
@@ -62,7 +62,10 @@ def timestamp_metadata(self):
     def test_year_2038_mtime_compilation(self):
         # Test to make sure we can handle mtimes larger than what a 32-bit
         # signed number can hold as part of bpo-34990
-        os.utime(self.source_path, (2**32 - 1, 2**32 - 1))
+        try:
+            os.utime(self.source_path, (2**32 - 1, 2**32 - 1))
+        except (OverflowError, OSError):
+            self.skipTest("filesystem doesn't support timestamps near 2**32")
         self.assertTrue(compileall.compile_file(self.source_path))
 
     def test_larger_than_32_bit_times(self):
diff --git a/Misc/NEWS.d/next/Library/2020-04-24-20-39-38.bpo-34990.3SmL9M.rst b/Misc/NEWS.d/next/Library/2020-04-24-20-39-38.bpo-34990.3SmL9M.rst
index 6a1f2ea88b08cf..d420b5dce1e3b5 100644
--- a/Misc/NEWS.d/next/Library/2020-04-24-20-39-38.bpo-34990.3SmL9M.rst
+++ b/Misc/NEWS.d/next/Library/2020-04-24-20-39-38.bpo-34990.3SmL9M.rst
@@ -1,2 +1,2 @@
 Fixed a Y2k38 bug in the compileall module where it would fail to compile
-files created after 2038.
+files with a modification time after the year 2038.