Issue #22599: Enhance tokenize.open() to be able to call it during Python

vstinner · vstinner · commit 969175091c45 · 2014-12-05T10:17:10.000+01:00
finalization.

Before the module kept a reference to the builtins module, but the module
attributes are cleared during Python finalization. Instead, keep directly a
reference to the open() function.

This enhancement is not perfect, calling tokenize.open() can still fail if
called very late during Python finalization.  Usually, the function is called
by the linecache module which is called to display a traceback or emit a
warning.
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
@@ -24,7 +24,6 @@
 __credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
                'Skip Montanaro, Raymond Hettinger, Trent Nelson, '
                'Michael Foord')
-import builtins
 from codecs import lookup, BOM_UTF8
 import collections
 from io import TextIOWrapper
@@ -430,11 +429,13 @@ def find_cookie(line):
     return default, [first, second]
 
 
+_builtin_open = open
+
 def open(filename):
     """Open a file in read only mode using the encoding detected by
     detect_encoding().
     """
-    buffer = builtins.open(filename, 'rb')
+    buffer = _builtin_open(filename, 'rb')
     encoding, lines = detect_encoding(buffer.readline)
     buffer.seek(0)
     text = TextIOWrapper(buffer, encoding, line_buffering=True)
@@ -657,7 +658,7 @@ def error(message, filename=None, location=None):
         # Tokenize the input
         if args.filename:
             filename = args.filename
-            with builtins.open(filename, 'rb') as f:
+            with _builtin_open(filename, 'rb') as f:
                 tokens = list(tokenize(f.readline))
         else:
             filename = "<stdin>"
