From 9b35bec6ca00895c48092619ad915b77e9c36cbb Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 23 Jul 2024 10:39:09 +0100 Subject: [PATCH 1/5] Move bound method deconstruction before monitoring call in INSTRUMENTED_CALL --- Include/internal/pycore_opcode_metadata.h | 6 +- Include/internal/pycore_uop_ids.h | 129 ++++++++-------- Include/internal/pycore_uop_metadata.h | 4 + Include/opcode_ids.h | 20 +-- Lib/_opcode_metadata.py | 20 +-- Python/bytecodes.c | 62 ++++---- Python/executor_cases.c.h | 34 ++++- Python/generated_cases.c.h | 167 ++++++++++++++++++--- Python/opcode_targets.h | 2 +- Python/optimizer_cases.c.h | 18 ++- Tools/cases_generator/analyzer.py | 5 +- Tools/cases_generator/generators_common.py | 7 +- Tools/cases_generator/stack.py | 16 +- 13 files changed, 336 insertions(+), 154 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 40e582a5e94c3b..f35d6312d16728 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -216,7 +216,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case IMPORT_NAME: return 2; case INSTRUMENTED_CALL: - return 0; + return 2 + oparg; case INSTRUMENTED_CALL_FUNCTION_EX: return 0; case INSTRUMENTED_CALL_KW: @@ -663,7 +663,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case IMPORT_NAME: return 1; case INSTRUMENTED_CALL: - return 0; + return 1; case INSTRUMENTED_CALL_FUNCTION_EX: return 0; case INSTRUMENTED_CALL_KW: @@ -1079,7 +1079,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, 0 }, [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index aa7ee7775faeba..43b8ebb7a6b98f 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -33,12 +33,11 @@ extern "C" { #define _BUILD_SLICE BUILD_SLICE #define _BUILD_STRING BUILD_STRING #define _BUILD_TUPLE BUILD_TUPLE -#define _CALL 311 #define _CALL_ALLOC_AND_ENTER_INIT CALL_ALLOC_AND_ENTER_INIT -#define _CALL_BUILTIN_CLASS 312 -#define _CALL_BUILTIN_FAST 313 -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 314 -#define _CALL_BUILTIN_O 315 +#define _CALL_BUILTIN_CLASS 311 +#define _CALL_BUILTIN_FAST 312 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 313 +#define _CALL_BUILTIN_O 314 #define _CALL_FUNCTION_EX CALL_FUNCTION_EX #define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 #define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 @@ -46,38 +45,38 @@ extern "C" { #define _CALL_KW CALL_KW #define _CALL_LEN CALL_LEN #define _CALL_LIST_APPEND CALL_LIST_APPEND -#define _CALL_METHOD_DESCRIPTOR_FAST 316 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 317 -#define _CALL_METHOD_DESCRIPTOR_NOARGS 318 -#define _CALL_METHOD_DESCRIPTOR_O 319 -#define _CALL_NON_PY_GENERAL 320 -#define _CALL_STR_1 321 -#define _CALL_TUPLE_1 322 +#define _CALL_METHOD_DESCRIPTOR_FAST 315 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 316 +#define _CALL_METHOD_DESCRIPTOR_NOARGS 317 +#define _CALL_METHOD_DESCRIPTOR_O 318 +#define _CALL_NON_PY_GENERAL 319 +#define _CALL_STR_1 320 +#define _CALL_TUPLE_1 321 #define _CALL_TYPE_1 CALL_TYPE_1 -#define _CHECK_ATTR_CLASS 323 -#define _CHECK_ATTR_METHOD_LAZY_DICT 324 -#define _CHECK_ATTR_MODULE 325 -#define _CHECK_ATTR_WITH_HINT 326 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 327 +#define _CHECK_ATTR_CLASS 322 +#define _CHECK_ATTR_METHOD_LAZY_DICT 323 +#define _CHECK_ATTR_MODULE 324 +#define _CHECK_ATTR_WITH_HINT 325 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 326 #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _CHECK_FUNCTION 328 -#define _CHECK_FUNCTION_EXACT_ARGS 329 -#define _CHECK_FUNCTION_VERSION 330 -#define _CHECK_IS_NOT_PY_CALLABLE 331 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 332 -#define _CHECK_METHOD_VERSION 333 -#define _CHECK_PEP_523 334 -#define _CHECK_PERIODIC 335 -#define _CHECK_STACK_SPACE 336 -#define _CHECK_STACK_SPACE_OPERAND 337 -#define _CHECK_VALIDITY 338 -#define _CHECK_VALIDITY_AND_SET_IP 339 -#define _COMPARE_OP 340 -#define _COMPARE_OP_FLOAT 341 -#define _COMPARE_OP_INT 342 -#define _COMPARE_OP_STR 343 -#define _CONTAINS_OP 344 +#define _CHECK_FUNCTION 327 +#define _CHECK_FUNCTION_EXACT_ARGS 328 +#define _CHECK_FUNCTION_VERSION 329 +#define _CHECK_IS_NOT_PY_CALLABLE 330 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 331 +#define _CHECK_METHOD_VERSION 332 +#define _CHECK_PEP_523 333 +#define _CHECK_PERIODIC 334 +#define _CHECK_STACK_SPACE 335 +#define _CHECK_STACK_SPACE_OPERAND 336 +#define _CHECK_VALIDITY 337 +#define _CHECK_VALIDITY_AND_SET_IP 338 +#define _COMPARE_OP 339 +#define _COMPARE_OP_FLOAT 340 +#define _COMPARE_OP_INT 341 +#define _COMPARE_OP_STR 342 +#define _CONTAINS_OP 343 #define _CONTAINS_OP_DICT CONTAINS_OP_DICT #define _CONTAINS_OP_SET CONTAINS_OP_SET #define _CONVERT_VALUE CONVERT_VALUE @@ -89,9 +88,10 @@ extern "C" { #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 345 +#define _DEOPT 344 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE +#define _DO_CALL 345 #define _DYNAMIC_EXIT 346 #define _END_SEND END_SEND #define _ERROR_POP_N 347 @@ -138,7 +138,6 @@ extern "C" { #define _INIT_CALL_PY_EXACT_ARGS_2 377 #define _INIT_CALL_PY_EXACT_ARGS_3 378 #define _INIT_CALL_PY_EXACT_ARGS_4 379 -#define _INSTRUMENTED_CALL INSTRUMENTED_CALL #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER @@ -225,53 +224,55 @@ extern "C" { #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE +#define _MAYBE_EXPAND_METHOD 425 +#define _MONITOR_CALL 426 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 425 -#define _POP_JUMP_IF_TRUE 426 +#define _POP_JUMP_IF_FALSE 427 +#define _POP_JUMP_IF_TRUE 428 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 427 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 429 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 428 +#define _PUSH_FRAME 430 #define _PUSH_NULL PUSH_NULL -#define _PY_FRAME_GENERAL 429 -#define _REPLACE_WITH_TRUE 430 +#define _PY_FRAME_GENERAL 431 +#define _REPLACE_WITH_TRUE 432 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 431 -#define _SEND 432 -#define _SEND_GEN_FRAME 433 +#define _SAVE_RETURN_OFFSET 433 +#define _SEND 434 +#define _SEND_GEN_FRAME 435 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 434 -#define _STORE_ATTR 435 -#define _STORE_ATTR_INSTANCE_VALUE 436 -#define _STORE_ATTR_SLOT 437 -#define _STORE_ATTR_WITH_HINT 438 +#define _START_EXECUTOR 436 +#define _STORE_ATTR 437 +#define _STORE_ATTR_INSTANCE_VALUE 438 +#define _STORE_ATTR_SLOT 439 +#define _STORE_ATTR_WITH_HINT 440 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 439 -#define _STORE_FAST_0 440 -#define _STORE_FAST_1 441 -#define _STORE_FAST_2 442 -#define _STORE_FAST_3 443 -#define _STORE_FAST_4 444 -#define _STORE_FAST_5 445 -#define _STORE_FAST_6 446 -#define _STORE_FAST_7 447 +#define _STORE_FAST 441 +#define _STORE_FAST_0 442 +#define _STORE_FAST_1 443 +#define _STORE_FAST_2 444 +#define _STORE_FAST_3 445 +#define _STORE_FAST_4 446 +#define _STORE_FAST_5 447 +#define _STORE_FAST_6 448 +#define _STORE_FAST_7 449 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 448 +#define _STORE_SUBSCR 450 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 449 -#define _TO_BOOL 450 +#define _TIER2_RESUME_CHECK 451 +#define _TO_BOOL 452 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -281,13 +282,13 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 451 +#define _UNPACK_SEQUENCE 453 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 451 +#define MAX_UOP_ID 453 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index ea48f9d20607bd..0ee8df2dd8ba89 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -198,6 +198,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = HAS_ARG_FLAG, [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_DEOPT_FLAG, [_LOAD_ATTR_METHOD_LAZY_DICT] = HAS_ARG_FLAG, + [_MAYBE_EXPAND_METHOD] = HAS_ARG_FLAG, [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG, [_PY_FRAME_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CHECK_FUNCTION_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, @@ -462,6 +463,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_MATCH_KEYS] = "_MATCH_KEYS", [_MATCH_MAPPING] = "_MATCH_MAPPING", [_MATCH_SEQUENCE] = "_MATCH_SEQUENCE", + [_MAYBE_EXPAND_METHOD] = "_MAYBE_EXPAND_METHOD", [_NOP] = "_NOP", [_POP_EXCEPT] = "_POP_EXCEPT", [_POP_TOP] = "_POP_TOP", @@ -884,6 +886,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _LOAD_ATTR_METHOD_LAZY_DICT: return 1; + case _MAYBE_EXPAND_METHOD: + return 2 + oparg; case _CHECK_PERIODIC: return 0; case _PY_FRAME_GENERAL: diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 2ae6e5c1ba51ec..730ae3dadf998e 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -210,16 +210,16 @@ extern "C" { #define INSTRUMENTED_YIELD_VALUE 241 #define INSTRUMENTED_LOAD_SUPER_ATTR 242 #define INSTRUMENTED_FOR_ITER 243 -#define INSTRUMENTED_CALL 244 -#define INSTRUMENTED_CALL_KW 245 -#define INSTRUMENTED_CALL_FUNCTION_EX 246 -#define INSTRUMENTED_INSTRUCTION 247 -#define INSTRUMENTED_JUMP_FORWARD 248 -#define INSTRUMENTED_JUMP_BACKWARD 249 -#define INSTRUMENTED_POP_JUMP_IF_TRUE 250 -#define INSTRUMENTED_POP_JUMP_IF_FALSE 251 -#define INSTRUMENTED_POP_JUMP_IF_NONE 252 -#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 253 +#define INSTRUMENTED_CALL_KW 244 +#define INSTRUMENTED_CALL_FUNCTION_EX 245 +#define INSTRUMENTED_INSTRUCTION 246 +#define INSTRUMENTED_JUMP_FORWARD 247 +#define INSTRUMENTED_JUMP_BACKWARD 248 +#define INSTRUMENTED_POP_JUMP_IF_TRUE 249 +#define INSTRUMENTED_POP_JUMP_IF_FALSE 250 +#define INSTRUMENTED_POP_JUMP_IF_NONE 251 +#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 252 +#define INSTRUMENTED_CALL 253 #define INSTRUMENTED_LINE 254 #define JUMP 256 #define JUMP_NO_INTERRUPT 257 diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index 2d0a5ba46b3e10..15021fdc463ee4 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -314,16 +314,16 @@ 'INSTRUMENTED_YIELD_VALUE': 241, 'INSTRUMENTED_LOAD_SUPER_ATTR': 242, 'INSTRUMENTED_FOR_ITER': 243, - 'INSTRUMENTED_CALL': 244, - 'INSTRUMENTED_CALL_KW': 245, - 'INSTRUMENTED_CALL_FUNCTION_EX': 246, - 'INSTRUMENTED_INSTRUCTION': 247, - 'INSTRUMENTED_JUMP_FORWARD': 248, - 'INSTRUMENTED_JUMP_BACKWARD': 249, - 'INSTRUMENTED_POP_JUMP_IF_TRUE': 250, - 'INSTRUMENTED_POP_JUMP_IF_FALSE': 251, - 'INSTRUMENTED_POP_JUMP_IF_NONE': 252, - 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 253, + 'INSTRUMENTED_CALL_KW': 244, + 'INSTRUMENTED_CALL_FUNCTION_EX': 245, + 'INSTRUMENTED_INSTRUCTION': 246, + 'INSTRUMENTED_JUMP_FORWARD': 247, + 'INSTRUMENTED_JUMP_BACKWARD': 248, + 'INSTRUMENTED_POP_JUMP_IF_TRUE': 249, + 'INSTRUMENTED_POP_JUMP_IF_FALSE': 250, + 'INSTRUMENTED_POP_JUMP_IF_NONE': 251, + 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 252, + 'INSTRUMENTED_CALL': 253, 'JUMP': 256, 'JUMP_NO_INTERRUPT': 257, 'LOAD_CLOSURE': 258, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 480045069c2942..37655659ad95cc 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3284,20 +3284,6 @@ dummy_func( unused/1 + _LOAD_ATTR_METHOD_LAZY_DICT; - inst(INSTRUMENTED_CALL, (unused/3 -- )) { - int is_meth = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 1)) != NULL; - int total_args = oparg + is_meth; - PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 2)); - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(PEEK(total_args)); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); - ERROR_IF(err, error); - PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(CALL); - } - // Cache layout: counter/1, func_version/2 // CALL_INTRINSIC_1/2, CALL_KW, and CALL_FUNCTION_EX aren't members! family(CALL, INLINE_CACHE_ENTRIES_CALL) = { @@ -3335,28 +3321,34 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION */ } + op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, args[oparg] -- func, maybe_self, args[oparg])) { + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + func = PyStackRef_FromPyObjectNew(method); + /* Make sure that callable and all args are in memory */ + args[-2] = func; + args[-1] = maybe_self; + PyStackRef_CLOSE(callable); + } + else { + func = callable; + maybe_self = self_or_null; + } + } + // When calling Python, inline the call using DISPATCH_INLINED(). - op(_CALL, (callable, self_or_null, args[oparg] -- res)) { + op(_DO_CALL, (callable, self_or_null, args[oparg] -- res)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null_o != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - else if (Py_TYPE(callable_o) == &PyMethod_Type) { - args--; - total_args++; - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - args[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - args[-1] = PyStackRef_FromPyObjectNew(method); - PyStackRef_CLOSE(callable); - callable_o = method; - callable = args[-1]; - } // Check if the call can be inlined or not if (Py_TYPE(callable_o) == &PyFunction_Type && tstate->interp->eval_frame == NULL && @@ -3419,7 +3411,19 @@ dummy_func( CHECK_EVAL_BREAKER(); } - macro(CALL) = _SPECIALIZE_CALL + unused/2 + _CALL + _CHECK_PERIODIC; + op(_MONITOR_CALL, (func, maybe_self, args[oparg] -- func, maybe_self, args[oparg])) { + int is_func = PyStackRef_IsNull(maybe_self); + PyObject *function = PyStackRef_AsPyObjectBorrow(func); + PyObject *arg = is_func ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(maybe_self); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); + ERROR_IF(err, error); + } + + macro(CALL) = _SPECIALIZE_CALL + unused/2 + _MAYBE_EXPAND_METHOD + _DO_CALL + _CHECK_PERIODIC; + macro(INSTRUMENTED_CALL) = unused/3 + _MAYBE_EXPAND_METHOD + _MONITOR_CALL + _DO_CALL + _CHECK_PERIODIC; op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e9f73f032bf2a4..6083436073707e 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3543,15 +3543,45 @@ break; } - /* _INSTRUMENTED_CALL is not a viable micro-op for tier 2 because it is instrumented */ + case _MAYBE_EXPAND_METHOD: { + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef func; + _PyStackRef maybe_self; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + func = PyStackRef_FromPyObjectNew(method); + /* Make sure that callable and all args are in memory */ + args[-2] = func; + args[-1] = maybe_self; + PyStackRef_CLOSE(callable); + } + else { + func = callable; + maybe_self = self_or_null; + } + stack_pointer[-2 - oparg] = func; + stack_pointer[-1 - oparg] = maybe_self; + break; + } - /* _CALL is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + /* _DO_CALL is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _CHECK_PERIODIC: { CHECK_EVAL_BREAKER(); break; } + /* _MONITOR_CALL is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + case _PY_FRAME_GENERAL: { _PyStackRef *args; _PyStackRef self_or_null; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 585e6825a346d8..f01d195abf0fb8 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -800,6 +800,8 @@ _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; + _PyStackRef func; + _PyStackRef maybe_self; _PyStackRef res; // _SPECIALIZE_CALL self_or_null = stack_pointer[-1 - oparg]; @@ -819,27 +821,35 @@ #endif /* ENABLE_SPECIALIZATION */ } /* Skip 2 cache entries */ - // _CALL + // _MAYBE_EXPAND_METHOD + { + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + func = PyStackRef_FromPyObjectNew(method); + /* Make sure that callable and all args are in memory */ + args[-2] = func; + args[-1] = maybe_self; + PyStackRef_CLOSE(callable); + } + else { + func = callable; + maybe_self = self_or_null; + } + } + // _DO_CALL + self_or_null = maybe_self; + callable = func; { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null_o != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - else if (Py_TYPE(callable_o) == &PyMethod_Type) { - args--; - total_args++; - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - args[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - args[-1] = PyStackRef_FromPyObjectNew(method); - PyStackRef_CLOSE(callable); - callable_o = method; - callable = args[-1]; - } // Check if the call can be inlined or not if (Py_TYPE(callable_o) == &PyFunction_Type && tstate->interp->eval_frame == NULL && @@ -3531,18 +3541,125 @@ (void)this_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef func; + _PyStackRef maybe_self; + _PyStackRef res; /* Skip 3 cache entries */ - int is_meth = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 1)) != NULL; - int total_args = oparg + is_meth; - PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 2)); - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(PEEK(total_args)); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); - if (err) goto error; - PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(CALL); + // _MAYBE_EXPAND_METHOD + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + func = PyStackRef_FromPyObjectNew(method); + /* Make sure that callable and all args are in memory */ + args[-2] = func; + args[-1] = maybe_self; + PyStackRef_CLOSE(callable); + } + else { + func = callable; + maybe_self = self_or_null; + } + } + // _MONITOR_CALL + { + int is_func = PyStackRef_IsNull(maybe_self); + PyObject *function = PyStackRef_AsPyObjectBorrow(func); + PyObject *arg = is_func ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(maybe_self); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); + if (err) goto error; + } + // _DO_CALL + self_or_null = maybe_self; + callable = func; + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + args, total_args, NULL + ); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + goto error; + } + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) { stack_pointer += -2 - oparg; goto error; } + } + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + if (res_o == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + if (err < 0) { + Py_CLEAR(res_o); + } + } + } + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CHECK_EVAL_BREAKER(); + DISPATCH(); } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 6097b249c0ad0b..3940472174a712 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -243,7 +243,6 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_YIELD_VALUE, &&TARGET_INSTRUMENTED_LOAD_SUPER_ATTR, &&TARGET_INSTRUMENTED_FOR_ITER, - &&TARGET_INSTRUMENTED_CALL, &&TARGET_INSTRUMENTED_CALL_KW, &&TARGET_INSTRUMENTED_CALL_FUNCTION_EX, &&TARGET_INSTRUMENTED_INSTRUCTION, @@ -253,6 +252,7 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_POP_JUMP_IF_FALSE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_NONE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + &&TARGET_INSTRUMENTED_CALL, &&TARGET_INSTRUMENTED_LINE, &&_unknown_opcode, }; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 60cfb214835bdd..d6c133be919b74 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1602,14 +1602,28 @@ break; } - /* _INSTRUMENTED_CALL is not a viable micro-op for tier 2 */ + case _MAYBE_EXPAND_METHOD: { + _Py_UopsSymbol *func; + _Py_UopsSymbol *maybe_self; + _Py_UopsSymbol **args; + func = sym_new_not_null(ctx); + maybe_self = sym_new_not_null(ctx); + for (int _i = oparg; --_i >= 0;) { + args[_i] = sym_new_not_null(ctx); + } + stack_pointer[-2 - oparg] = func; + stack_pointer[-1 - oparg] = maybe_self; + break; + } - /* _CALL is not a viable micro-op for tier 2 */ + /* _DO_CALL is not a viable micro-op for tier 2 */ case _CHECK_PERIODIC: { break; } + /* _MONITOR_CALL is not a viable micro-op for tier 2 */ + case _PY_FRAME_GENERAL: { _Py_UopsSymbol **args; _Py_UopsSymbol *self_or_null; diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index c6f044e066539e..2a8909e32feeb9 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -317,9 +317,12 @@ def analyze_stack(op: parser.InstDef | parser.Pseudo, replace_op_arg_1: str | No convert_stack_item(i, replace_op_arg_1) for i in op.inputs if isinstance(i, parser.StackEffect) ] outputs: list[StackItem] = [convert_stack_item(i, replace_op_arg_1) for i in op.outputs] + modified = False for input, output in zip(inputs, outputs): - if input.name == output.name: + if input.name == output.name and not modified: input.peek = output.peek = True + else: + modified = True if isinstance(op, parser.InstDef): output_names = [out.name for out in outputs] for input in inputs: diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 9314bb9e79687f..0722626978e97c 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -92,9 +92,9 @@ def replace_error( next(tkn_iter) # RPAREN next(tkn_iter) # Semi colon out.emit(") ") - c_offset = stack.peek_offset.to_c() + c_offset = stack.peek_offset() try: - offset = -int(c_offset) + offset = int(c_offset) close = ";\n" except ValueError: offset = None @@ -102,7 +102,8 @@ def replace_error( close = "; }\n" out.emit("goto ") if offset: - out.emit(f"pop_{offset}_") + assert offset < 0 + out.emit(f"pop_{-offset}_") out.emit(label) out.emit(close) diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index f497fa34dfced4..fe3c877bfc1449 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -49,6 +49,9 @@ class StackOffset: def empty() -> "StackOffset": return StackOffset([], []) + def copy(self): + return StackOffset(self.popped[:], self.pushed[:]) + def pop(self, item: StackItem) -> None: self.popped.append(var_size(item)) @@ -122,14 +125,12 @@ class Stack: def __init__(self) -> None: self.top_offset = StackOffset.empty() self.base_offset = StackOffset.empty() - self.peek_offset = StackOffset.empty() + self.variables: list[StackItem] = [] self.defined: set[str] = set() def pop(self, var: StackItem, extract_bits: bool = False) -> str: self.top_offset.pop(var) - if not var.peek: - self.peek_offset.pop(var) indirect = "&" if var.is_array() else "" if self.variables: popped = self.variables.pop() @@ -210,9 +211,16 @@ def flush(self, out: CWriter, cast_type: str = "uintptr_t", extract_bits: bool = self.variables = [] self.base_offset.clear() self.top_offset.clear() - self.peek_offset.clear() out.start_line() + def peek_offset(self): + peek = self.base_offset.copy() + for var in self.variables: + if not var.peek: + break + peek.push(var) + return peek.to_c() + def as_comment(self) -> str: return f"/* Variables: {[v.name for v in self.variables]}. Base offset: {self.base_offset.to_c()}. Top offset: {self.top_offset.to_c()} */" From 5787d3ea77d073a8801b7911046677a1427a9dd6 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 23 Jul 2024 14:32:51 +0100 Subject: [PATCH 2/5] Revert unnecessary change --- Tools/cases_generator/generators_common.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 0722626978e97c..587dc0d03eded5 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -94,7 +94,7 @@ def replace_error( out.emit(") ") c_offset = stack.peek_offset() try: - offset = int(c_offset) + offset = -int(c_offset) close = ";\n" except ValueError: offset = None @@ -102,8 +102,7 @@ def replace_error( close = "; }\n" out.emit("goto ") if offset: - assert offset < 0 - out.emit(f"pop_{-offset}_") + out.emit(f"pop_{offset}_") out.emit(label) out.emit(close) From 5adceac70579fece2bd36de4f212a4488344392e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 24 Jul 2024 11:15:07 +0100 Subject: [PATCH 3/5] Fix zero-th argument when monitoring --- Python/bytecodes.c | 19 +++++++++++++++---- Python/generated_cases.c.h | 17 ++++++++++++++--- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 37655659ad95cc..ec781d003c8dc7 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3414,11 +3414,22 @@ dummy_func( op(_MONITOR_CALL, (func, maybe_self, args[oparg] -- func, maybe_self, args[oparg])) { int is_func = PyStackRef_IsNull(maybe_self); PyObject *function = PyStackRef_AsPyObjectBorrow(func); - PyObject *arg = is_func ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(maybe_self); + PyObject *arg0; + if (is_func) { + if (oparg) { + arg0 = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg0 = &_PyInstrumentation_MISSING; + } + } + else { + arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); + } int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg0 + ); ERROR_IF(err, error); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index f01d195abf0fb8..1b29637badb91f 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3573,11 +3573,22 @@ { int is_func = PyStackRef_IsNull(maybe_self); PyObject *function = PyStackRef_AsPyObjectBorrow(func); - PyObject *arg = is_func ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(maybe_self); + PyObject *arg0; + if (is_func) { + if (oparg) { + arg0 = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg0 = &_PyInstrumentation_MISSING; + } + } + else { + arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); + } int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); + frame, this_instr, function, arg0 + ); if (err) goto error; } // _DO_CALL From 72e1fcf1e4cb8183472600cdc7f1b537db24db1c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 24 Jul 2024 11:15:23 +0100 Subject: [PATCH 4/5] Update monitoring test --- Lib/test/test_monitoring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index a07be306986b43..1a129b9432e72d 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -1575,7 +1575,7 @@ def f(): ('line', 'method', 2), ('line', 'method', 3), ('line', 'method', 2), - ('call', 'method', 1), + ('call', 'method', d["b"]), ('line', 'method', 1), ('line', 'method', 1), ('line', 'get_events', 11), From 2824cb51a1bef285552d3034392fe889daa48c7b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 26 Jul 2024 12:35:45 +0100 Subject: [PATCH 5/5] Tidy up _MONITOR_CALL --- Python/bytecodes.c | 16 +++++++--------- Python/generated_cases.c.h | 16 +++++++--------- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8e210b55451869..871e2dbf358418 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3369,19 +3369,17 @@ dummy_func( } op(_MONITOR_CALL, (func, maybe_self, args[oparg] -- func, maybe_self, args[oparg])) { - int is_func = PyStackRef_IsNull(maybe_self); + int is_meth = !PyStackRef_IsNull(maybe_self); PyObject *function = PyStackRef_AsPyObjectBorrow(func); PyObject *arg0; - if (is_func) { - if (oparg) { - arg0 = PyStackRef_AsPyObjectBorrow(args[0]); - } - else { - arg0 = &_PyInstrumentation_MISSING; - } + if (is_meth) { + arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); + } + else if (oparg) { + arg0 = PyStackRef_AsPyObjectBorrow(args[0]); } else { - arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); + arg0 = &_PyInstrumentation_MISSING; } int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index d6d232ee763773..76d1cc7ad6cf95 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3544,19 +3544,17 @@ } // _MONITOR_CALL { - int is_func = PyStackRef_IsNull(maybe_self); + int is_meth = !PyStackRef_IsNull(maybe_self); PyObject *function = PyStackRef_AsPyObjectBorrow(func); PyObject *arg0; - if (is_func) { - if (oparg) { - arg0 = PyStackRef_AsPyObjectBorrow(args[0]); - } - else { - arg0 = &_PyInstrumentation_MISSING; - } + if (is_meth) { + arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); + } + else if (oparg) { + arg0 = PyStackRef_AsPyObjectBorrow(args[0]); } else { - arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); + arg0 = &_PyInstrumentation_MISSING; } int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL,