gh-92031: Deoptimize Static Code at Finalization (GH-92039)

sweeneyde · web-flow · commit b156578bd63a · 2022-05-03T08:59:12.000-06:00
diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h
diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
@@ -343,6 +343,20 @@ def test_finalize_structseq(self):
         out, err = self.run_embedded_interpreter("test_repeated_init_exec", code)
         self.assertEqual(out, 'Tests passed\n' * INIT_LOOPS)
 
+    @support.skip_if_pgo_task
+    def test_quickened_static_code_gets_unquickened_at_Py_FINALIZE(self):
+        # https://github.com/python/cpython/issues/92031
+        code = """if 1:
+            from importlib._bootstrap import _handle_fromlist
+            import dis
+            for name in dis.opmap:
+                # quicken this frozen code object.
+                _handle_fromlist(dis, [name], lambda *args: None)
+        """
+        run = self.run_embedded_interpreter
+        for i in range(50):
+            out, err = run("test_repeated_init_exec", code, timeout=60)
+
     def test_ucnhash_capi_reset(self):
         # bpo-47182: unicodeobject.c:ucnhash_capi was not reset on shutdown.
         code = "print('\\N{digit nine}')"
diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-04-29-02-50-41.gh-issue-92031.2PpaIN.rst b/Misc/NEWS.d/next/Core and Builtins/2022-04-29-02-50-41.gh-issue-92031.2PpaIN.rst
@@ -0,0 +1 @@
+Deoptimize statically-allocated code objects during ``Py_FINALIZE()`` so that future ``_PyCode_Quicken`` calls always start with unquickened code.
diff --git a/Objects/codeobject.c b/Objects/codeobject.c
@@ -1350,23 +1350,29 @@ _PyCode_GetFreevars(PyCodeObject *co)
     return get_localsplus_names(co, CO_FAST_FREE, co->co_nfreevars);
 }
 
-PyObject *
-_PyCode_GetCode(PyCodeObject *co)
+static void
+deopt_code(_Py_CODEUNIT *instructions, Py_ssize_t len)
 {
-    PyObject *code = PyBytes_FromStringAndSize(NULL, _PyCode_NBYTES(co));
-    if (code == NULL) {
-        return NULL;
-    }
-    _Py_CODEUNIT *instructions = (_Py_CODEUNIT *)PyBytes_AS_STRING(code);
-    for (int i = 0; i < Py_SIZE(co); i++) {
-        _Py_CODEUNIT instruction = _PyCode_CODE(co)[i];
-        int opcode = _PyOpcode_Deopt[_Py_OPCODE(instruction)];
+    for (int i = 0; i < len; i++) {
+        _Py_CODEUNIT instruction = instructions[i];
+        int opcode = _PyOpcode_Original[_Py_OPCODE(instruction)];
         int caches = _PyOpcode_Caches[opcode];
         instructions[i] = _Py_MAKECODEUNIT(opcode, _Py_OPARG(instruction));
         while (caches--) {
             instructions[++i] = _Py_MAKECODEUNIT(CACHE, 0);
         }
     }
+}
+
+PyObject *
+_PyCode_GetCode(PyCodeObject *co)
+{
+    PyObject *code = PyBytes_FromStringAndSize((const char *)_PyCode_CODE(co),
+                                               _PyCode_NBYTES(co));
+    if (code == NULL) {
+        return NULL;
+    }
+    deopt_code((_Py_CODEUNIT *)PyBytes_AS_STRING(code), Py_SIZE(co));
     return code;
 }
 
@@ -2076,6 +2082,7 @@ _PyStaticCode_Dealloc(PyCodeObject *co)
     if (co->co_warmup == 0) {
          _Py_QuickenedCount--;
     }
+    deopt_code(_PyCode_CODE(co), Py_SIZE(co));
     co->co_warmup = QUICKENING_INITIAL_WARMUP_VALUE;
     PyMem_Free(co->co_extra);
     co->co_extra = NULL;
diff --git a/Tools/scripts/generate_opcode_h.py b/Tools/scripts/generate_opcode_h.py
@@ -117,6 +117,7 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna
 
         iobj.write("\nextern const uint8_t _PyOpcode_Caches[256];\n")
         iobj.write("\nextern const uint8_t _PyOpcode_Deopt[256];\n")
+        iobj.write("\nextern const uint8_t _PyOpcode_Original[256];\n")
         iobj.write("\n#ifdef NEED_OPCODE_TABLES\n")
         write_int_array_from_ops("_PyOpcode_RelativeJump", opcode['hasjrel'], iobj)
         write_int_array_from_ops("_PyOpcode_Jump", opcode['hasjrel'] + opcode['hasjabs'], iobj)
@@ -137,6 +138,12 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna
         for opt, deopt in sorted(deoptcodes.items()):
             iobj.write(f"    [{opt}] = {deopt},\n")
         iobj.write("};\n")
+        iobj.write("\nconst uint8_t _PyOpcode_Original[256] = {\n")
+        for opt, deopt in sorted(deoptcodes.items()):
+            if opt.startswith("EXTENDED_ARG"):
+                deopt = "EXTENDED_ARG_QUICK"
+            iobj.write(f"    [{opt}] = {deopt},\n")
+        iobj.write("};\n")
         iobj.write("#endif   // NEED_OPCODE_TABLES\n")
 
         fobj.write("\n")

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+Deoptimize statically-allocated code objects during ``Py_FINALIZE()`` so that future ``_PyCode_Quicken`` calls always start with unquickened code.