From d267140ea772b8578864b0c11faa5df4f80e386b Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Fri, 22 Mar 2024 17:26:12 +0000 Subject: [PATCH 1/6] gh-112383: teach dis how to interpret ENTER_EXECUTOR --- Lib/dis.py | 26 +++++++++++++++++++++----- Lib/test/test_dis.py | 11 +---------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/Lib/dis.py b/Lib/dis.py index d146bcbb5097ef..d6ca26358d66d7 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -204,8 +204,27 @@ def _deoptop(op): name = _all_opname[op] return _all_opmap[deoptmap[name]] if name in deoptmap else op +import _testinternalcapi def _get_code_array(co, adaptive): - return co._co_code_adaptive if adaptive else co.co_code + if adaptive: + code = co._co_code_adaptive + res = [] + found = False + for i in range(0, len(code), 2): + op, arg = code[i], code[i+1] + if op == ENTER_EXECUTOR: + try: + ex = _testinternalcapi.get_executor(co, i) + except ValueError: + ex = None + if ex: + op, arg = ex.get_opcode(), ex.get_oparg() + found = True + res.append(op.to_bytes()) + res.append(arg.to_bytes()) + return code if not found else b''.join(res) + else: + return co.co_code def code_info(x): """Formatted details of methods, functions, or code.""" @@ -514,8 +533,6 @@ def offset_from_jump_arg(self, op, arg, offset): argval = offset + 2 + signed_arg*2 caches = _get_cache_size(_all_opname[deop]) argval += 2 * caches - if deop == ENTER_EXECUTOR: - argval += 2 return argval return None @@ -680,8 +697,7 @@ def _parse_exception_table(code): def _is_backward_jump(op): return opname[op] in ('JUMP_BACKWARD', - 'JUMP_BACKWARD_NO_INTERRUPT', - 'ENTER_EXECUTOR') + 'JUMP_BACKWARD_NO_INTERRUPT') def _get_instructions_bytes(code, linestarts=None, line_offset=0, co_positions=None, original_code=None, arg_resolver=None): diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index a93cb509b651c5..747a73829fa705 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -1201,19 +1201,10 @@ def test_call_specialize(self): @cpython_only @requires_specialization def test_loop_quicken(self): - import _testinternalcapi # Loop can trigger a quicken where the loop is located - self.code_quicken(loop_test, 1) + self.code_quicken(loop_test, 4) got = self.get_disassembly(loop_test, adaptive=True) expected = dis_loop_test_quickened_code - if _testinternalcapi.get_optimizer(): - # We *may* see ENTER_EXECUTOR in the disassembly. This is a - # temporary hack to keep the test working until dis is able to - # handle the instruction correctly (GH-112383): - got = got.replace( - "ENTER_EXECUTOR 16", - "JUMP_BACKWARD 16 (to L1)", - ) self.do_disassembly_compare(got, expected) @cpython_only From 5b9a6284222904dd21e2bbe4445b6f68665577b2 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sat, 23 Mar 2024 13:32:08 +0000 Subject: [PATCH 2/6] move get_executor from _testinternalcapi to _opcode --- Lib/dis.py | 7 ++-- Lib/test/test_capi/test_opt.py | 22 ++++++------ Modules/_opcode.c | 21 ++++++++++++ Modules/_testinternalcapi.c | 21 ------------ Modules/clinic/_opcode.c.h | 62 +++++++++++++++++++++++++++++++++- 5 files changed, 97 insertions(+), 36 deletions(-) diff --git a/Lib/dis.py b/Lib/dis.py index d6ca26358d66d7..111d624fc259c5 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -17,6 +17,8 @@ _specialized_opmap, ) +from _opcode import get_executor + __all__ = ["code_info", "dis", "disassemble", "distb", "disco", "findlinestarts", "findlabels", "show_code", "get_instructions", "Instruction", "Bytecode"] + _opcodes_all @@ -204,7 +206,6 @@ def _deoptop(op): name = _all_opname[op] return _all_opmap[deoptmap[name]] if name in deoptmap else op -import _testinternalcapi def _get_code_array(co, adaptive): if adaptive: code = co._co_code_adaptive @@ -214,12 +215,14 @@ def _get_code_array(co, adaptive): op, arg = code[i], code[i+1] if op == ENTER_EXECUTOR: try: - ex = _testinternalcapi.get_executor(co, i) + ex = get_executor(co, i) except ValueError: ex = None + if ex: op, arg = ex.get_opcode(), ex.get_oparg() found = True + res.append(op.to_bytes()) res.append(arg.to_bytes()) return code if not found else b''.join(res) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index b0859a382de523..6cbdc410336579 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -6,6 +6,7 @@ import gc import os +import _opcode import _testinternalcapi from test.support import script_helper, requires_specialization @@ -115,13 +116,11 @@ def testfunc(x): def get_first_executor(func): code = func.__code__ co_code = code.co_code - JUMP_BACKWARD = opcode.opmap["JUMP_BACKWARD"] for i in range(0, len(co_code), 2): - if co_code[i] == JUMP_BACKWARD: - try: - return _testinternalcapi.get_executor(code, i) - except ValueError: - pass + try: + return _opcode.get_executor(code, i) + except ValueError: + pass return None @@ -760,17 +759,16 @@ def test_promote_globals_to_constants(self): result = script_helper.run_python_until_end('-c', textwrap.dedent(""" import _testinternalcapi import opcode + import _opcode def get_first_executor(func): code = func.__code__ co_code = code.co_code - JUMP_BACKWARD = opcode.opmap["JUMP_BACKWARD"] for i in range(0, len(co_code), 2): - if co_code[i] == JUMP_BACKWARD: - try: - return _testinternalcapi.get_executor(code, i) - except ValueError: - pass + try: + return _opcode.get_executor(code, i) + except ValueError: + pass return None def get_opnames(ex): diff --git a/Modules/_opcode.c b/Modules/_opcode.c index 93c71377f03a76..02a090a4b933a1 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -347,6 +347,26 @@ _opcode_get_intrinsic2_descs_impl(PyObject *module) return list; } +/*[clinic input] + +_opcode.get_executor + + code: object + offset: int + +Return the executor object at offset in code if exists, None otherwise. +[clinic start generated code]*/ + +static PyObject * +_opcode_get_executor_impl(PyObject *module, PyObject *code, int offset) +/*[clinic end generated code: output=c035c7a47b16648f input=85eff93ea7aac282]*/ +{ + if (!PyCode_Check(code)) { + PyErr_SetString(PyExc_TypeError, "first argument must be a code object"); + return NULL; + } + return (PyObject *)PyUnstable_GetExecutor((PyCodeObject *)code, offset); +} static PyMethodDef opcode_functions[] = { @@ -363,6 +383,7 @@ opcode_functions[] = { _OPCODE_GET_NB_OPS_METHODDEF _OPCODE_GET_INTRINSIC1_DESCS_METHODDEF _OPCODE_GET_INTRINSIC2_DESCS_METHODDEF + _OPCODE_GET_EXECUTOR_METHODDEF {NULL, NULL, 0, NULL} }; diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index e1717f7a66b1de..c07652facc0ae2 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -991,26 +991,6 @@ get_optimizer(PyObject *self, PyObject *Py_UNUSED(ignored)) return opt; } -static PyObject * -get_executor(PyObject *self, PyObject *const *args, Py_ssize_t nargs) -{ - - if (!_PyArg_CheckPositional("get_executor", nargs, 2, 2)) { - return NULL; - } - PyObject *code = args[0]; - PyObject *offset = args[1]; - long ioffset = PyLong_AsLong(offset); - if (ioffset == -1 && PyErr_Occurred()) { - return NULL; - } - if (!PyCode_Check(code)) { - PyErr_SetString(PyExc_TypeError, "first argument must be a code object"); - return NULL; - } - return (PyObject *)PyUnstable_GetExecutor((PyCodeObject *)code, ioffset); -} - static PyObject * add_executor_dependency(PyObject *self, PyObject *args) { @@ -1836,7 +1816,6 @@ static PyMethodDef module_functions[] = { {"iframe_getlasti", iframe_getlasti, METH_O, NULL}, {"get_optimizer", get_optimizer, METH_NOARGS, NULL}, {"set_optimizer", set_optimizer, METH_O, NULL}, - {"get_executor", _PyCFunction_CAST(get_executor), METH_FASTCALL, NULL}, {"new_counter_optimizer", new_counter_optimizer, METH_NOARGS, NULL}, {"new_uop_optimizer", new_uop_optimizer, METH_NOARGS, NULL}, {"add_executor_dependency", add_executor_dependency, METH_VARARGS, NULL}, diff --git a/Modules/clinic/_opcode.c.h b/Modules/clinic/_opcode.c.h index c7fd0f9f8a7420..fb90fb8e32f918 100644 --- a/Modules/clinic/_opcode.c.h +++ b/Modules/clinic/_opcode.c.h @@ -668,4 +668,64 @@ _opcode_get_intrinsic2_descs(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _opcode_get_intrinsic2_descs_impl(module); } -/*[clinic end generated code: output=a1052bb1deffb7f2 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_opcode_get_executor__doc__, +"get_executor($module, /, code, offset)\n" +"--\n" +"\n" +"Return the executor object at offset in code if exists, None otherwise."); + +#define _OPCODE_GET_EXECUTOR_METHODDEF \ + {"get_executor", _PyCFunction_CAST(_opcode_get_executor), METH_FASTCALL|METH_KEYWORDS, _opcode_get_executor__doc__}, + +static PyObject * +_opcode_get_executor_impl(PyObject *module, PyObject *code, int offset); + +static PyObject * +_opcode_get_executor(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(code), &_Py_ID(offset), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"code", "offset", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "get_executor", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *code; + int offset; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + code = args[0]; + offset = PyLong_AsInt(args[1]); + if (offset == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = _opcode_get_executor_impl(module, code, offset); + +exit: + return return_value; +} +/*[clinic end generated code: output=2dbb31b041b49c8f input=a9049054013a1b77]*/ From 768f6ca7d8fa2d86ba80a1bebab833126e673905 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sat, 23 Mar 2024 13:38:48 +0000 Subject: [PATCH 3/6] fix lint --- Lib/test/test_capi/test_opt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 6cbdc410336579..2c4fe890291c8d 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1,5 +1,4 @@ import contextlib -import opcode import sys import textwrap import unittest From 02769d73926fe410af4489e7c20b6ba69249320e Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sat, 23 Mar 2024 13:40:18 +0000 Subject: [PATCH 4/6] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2024-03-23-13-40-13.gh-issue-112383.XuHf3G.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2024-03-23-13-40-13.gh-issue-112383.XuHf3G.rst diff --git a/Misc/NEWS.d/next/Library/2024-03-23-13-40-13.gh-issue-112383.XuHf3G.rst b/Misc/NEWS.d/next/Library/2024-03-23-13-40-13.gh-issue-112383.XuHf3G.rst new file mode 100644 index 00000000000000..c99b0fdfed110d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-23-13-40-13.gh-issue-112383.XuHf3G.rst @@ -0,0 +1 @@ +Fix :mod:`dis` module's handling of `ENTER_EXECUTOR` instructions. From 7623c4fb48f26496e3b4f038bdd8bb778fd95e68 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sat, 23 Mar 2024 13:52:14 +0000 Subject: [PATCH 5/6] fix news --- .../next/Library/2024-03-23-13-40-13.gh-issue-112383.XuHf3G.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2024-03-23-13-40-13.gh-issue-112383.XuHf3G.rst b/Misc/NEWS.d/next/Library/2024-03-23-13-40-13.gh-issue-112383.XuHf3G.rst index c99b0fdfed110d..931e615c2b86c5 100644 --- a/Misc/NEWS.d/next/Library/2024-03-23-13-40-13.gh-issue-112383.XuHf3G.rst +++ b/Misc/NEWS.d/next/Library/2024-03-23-13-40-13.gh-issue-112383.XuHf3G.rst @@ -1 +1 @@ -Fix :mod:`dis` module's handling of `ENTER_EXECUTOR` instructions. +Fix :mod:`dis` module's handling of ``ENTER_EXECUTOR`` instructions. From 15bc15ec8331ab0ade19dcd4ed69e4665b7542d1 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sat, 23 Mar 2024 22:04:38 +0000 Subject: [PATCH 6/6] tweak exception message --- Modules/_opcode.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Modules/_opcode.c b/Modules/_opcode.c index 02a090a4b933a1..5350adb456b859 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -362,7 +362,9 @@ _opcode_get_executor_impl(PyObject *module, PyObject *code, int offset) /*[clinic end generated code: output=c035c7a47b16648f input=85eff93ea7aac282]*/ { if (!PyCode_Check(code)) { - PyErr_SetString(PyExc_TypeError, "first argument must be a code object"); + PyErr_Format(PyExc_TypeError, + "expected a code object, not '%.100s'", + Py_TYPE(code)->tp_name); return NULL; } return (PyObject *)PyUnstable_GetExecutor((PyCodeObject *)code, offset);