GH-95113: Don't use EXTENDED_ARG_QUICK in unquickened code (GH-95121)

brandtbucher · web-flow · commit e402b26b7fb9 · 2022-07-22T11:04:20.000-07:00
diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h
diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py
@@ -744,7 +744,7 @@ def extended_arg_quick():
 %3d           0 RESUME                   0
 
 %3d           2 LOAD_CONST               1 (Ellipsis)
-              4 EXTENDED_ARG_QUICK       1
+              4 EXTENDED_ARG             1
               6 UNPACK_EX              256
               8 STORE_FAST               0 (_)
              10 STORE_FAST               0 (_)
diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-07-21-17-54-52.gh-issue-95113.NnSLpT.rst b/Misc/NEWS.d/next/Core and Builtins/2022-07-21-17-54-52.gh-issue-95113.NnSLpT.rst
@@ -0,0 +1,4 @@
+Replace all ``EXTENDED_ARG_QUICK`` instructions with basic
+:opcode:`EXTENDED_ARG` instructions in unquickened code. Consumers of
+non-adaptive bytecode should be able to handle extended arguments the same
+way they were handled in CPython 3.10 and older.
diff --git a/Objects/codeobject.c b/Objects/codeobject.c
@@ -1418,7 +1418,7 @@ deopt_code(_Py_CODEUNIT *instructions, Py_ssize_t len)
 {
     for (int i = 0; i < len; i++) {
         _Py_CODEUNIT instruction = instructions[i];
-        int opcode = _PyOpcode_Original[_Py_OPCODE(instruction)];
+        int opcode = _PyOpcode_Deopt[_Py_OPCODE(instruction)];
         int caches = _PyOpcode_Caches[opcode];
         instructions[i] = _Py_MAKECODEUNIT(opcode, _Py_OPARG(instruction));
         while (caches--) {
diff --git a/Python/ceval.c b/Python/ceval.c
@@ -5631,8 +5631,16 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
             assert(oparg);
             oparg <<= 8;
             oparg |= _Py_OPARG(*next_instr);
+            // We might be tracing. To avoid breaking tracing guarantees in 
+            // quickened instructions, always deoptimize the next opcode:
             opcode = _PyOpcode_Deopt[_Py_OPCODE(*next_instr)];
             PRE_DISPATCH_GOTO();
+            // CPython hasn't traced the following instruction historically
+            // (DO_TRACING would clobber our extended oparg anyways), so just
+            // skip our usual cframe.use_tracing check before dispatch. Also,
+            // make sure the next instruction isn't a RESUME, since that needs
+            // to trace properly (and shouldn't have an extended arg anyways):
+            assert(opcode != RESUME);
             DISPATCH_GOTO();
         }
 
diff --git a/Python/compile.c b/Python/compile.c
@@ -211,13 +211,13 @@ write_instr(_Py_CODEUNIT *codestr, struct instr *instruction, int ilen)
     int caches = _PyOpcode_Caches[opcode];
     switch (ilen - caches) {
         case 4:
-            *codestr++ = _Py_MAKECODEUNIT(EXTENDED_ARG_QUICK, (oparg >> 24) & 0xFF);
+            *codestr++ = _Py_MAKECODEUNIT(EXTENDED_ARG, (oparg >> 24) & 0xFF);
             /* fall through */
         case 3:
-            *codestr++ = _Py_MAKECODEUNIT(EXTENDED_ARG_QUICK, (oparg >> 16) & 0xFF);
+            *codestr++ = _Py_MAKECODEUNIT(EXTENDED_ARG, (oparg >> 16) & 0xFF);
             /* fall through */
         case 2:
-            *codestr++ = _Py_MAKECODEUNIT(EXTENDED_ARG_QUICK, (oparg >> 8) & 0xFF);
+            *codestr++ = _Py_MAKECODEUNIT(EXTENDED_ARG, (oparg >> 8) & 0xFF);
             /* fall through */
         case 1:
             *codestr++ = _Py_MAKECODEUNIT(opcode, oparg & 0xFF);
diff --git a/Python/specialize.c b/Python/specialize.c
@@ -295,6 +295,9 @@ _PyCode_Quicken(PyCodeObject *code)
         else {
             assert(!_PyOpcode_Caches[opcode]);
             switch (opcode) {
+                case EXTENDED_ARG:
+                    _Py_SET_OPCODE(instructions[i], EXTENDED_ARG_QUICK);
+                    break;
                 case JUMP_BACKWARD:
                     _Py_SET_OPCODE(instructions[i], JUMP_BACKWARD_QUICK);
                     break;
diff --git a/Tools/scripts/generate_opcode_h.py b/Tools/scripts/generate_opcode_h.py
@@ -131,7 +131,6 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna
 
         iobj.write("\nextern const uint8_t _PyOpcode_Caches[256];\n")
         iobj.write("\nextern const uint8_t _PyOpcode_Deopt[256];\n")
-        iobj.write("\nextern const uint8_t _PyOpcode_Original[256];\n")
         iobj.write("\n#ifdef NEED_OPCODE_TABLES\n")
         write_int_array_from_ops("_PyOpcode_RelativeJump", opcode['hasjrel'], iobj)
         write_int_array_from_ops("_PyOpcode_Jump", opcode['hasjrel'] + opcode['hasjabs'], iobj)
@@ -153,12 +152,6 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna
         for opt, deopt in sorted(deoptcodes.items()):
             iobj.write(f"    [{opt}] = {deopt},\n")
         iobj.write("};\n")
-        iobj.write("\nconst uint8_t _PyOpcode_Original[256] = {\n")
-        for opt, deopt in sorted(deoptcodes.items()):
-            if opt.startswith("EXTENDED_ARG"):
-                deopt = "EXTENDED_ARG_QUICK"
-            iobj.write(f"    [{opt}] = {deopt},\n")
-        iobj.write("};\n")
         iobj.write("#endif   // NEED_OPCODE_TABLES\n")
 
         fobj.write("\n")

Original file line number	Diff line number	Diff line change
`@@ -1418,7 +1418,7 @@ deopt_code(_Py_CODEUNIT *instructions, Py_ssize_t len)`
`1418`	`1418`	`{`
`1419`	`1419`	`for (int i = 0; i < len; i++) {`
`1420`	`1420`	`_Py_CODEUNIT instruction = instructions[i];`
`1421`		`- int opcode = _PyOpcode_Original[_Py_OPCODE(instruction)];`
	`1421`	`+ int opcode = _PyOpcode_Deopt[_Py_OPCODE(instruction)];`
`1422`	`1422`	`int caches = _PyOpcode_Caches[opcode];`
`1423`	`1423`	`instructions[i] = _Py_MAKECODEUNIT(opcode, _Py_OPARG(instruction));`
`1424`	`1424`	`while (caches--) {`