From 234b8762c1bb67b31c553eb6edc9f4e9a128f8b8 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 3 Feb 2023 10:51:27 -0800 Subject: [PATCH 01/39] Special-case CHECK_EVAL_BREAKER() --- Tools/cases_generator/generate_cases.py | 15 ++++++++++++--- Tools/cases_generator/test_generator.py | 9 +++++++-- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 3925583b40e728..1f16aee373041e 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -230,7 +230,8 @@ def __init__(self, inst: parser.InstDef): self.kind = inst.kind self.name = inst.name self.block = inst.block - self.block_text, self.predictions = extract_block_text(self.block) + self.block_text, self.check_eval_breaker, self.predictions = \ + extract_block_text(self.block) self.always_exits = always_exits(self.block_text) self.cache_effects = [ effect for effect in inst.inputs if isinstance(effect, parser.CacheEffect) @@ -1016,6 +1017,8 @@ def write_instr(self, instr: Instruction) -> None: if not instr.always_exits: for prediction in instr.predictions: self.out.emit(f"PREDICT({prediction});") + if instr.check_eval_breaker: + self.out.emit("CHECK_EVAL_BREAKER();") self.out.emit(f"DISPATCH();") def write_super(self, sup: SuperInstruction) -> None: @@ -1091,7 +1094,7 @@ def wrap_super_or_macro(self, up: SuperOrMacroInstruction): self.out.emit(f"DISPATCH();") -def extract_block_text(block: parser.Block) -> tuple[list[str], list[str]]: +def extract_block_text(block: parser.Block) -> tuple[list[str], bool, list[str]]: # Get lines of text with proper dedent blocklines = block.text.splitlines(True) @@ -1111,6 +1114,12 @@ def extract_block_text(block: parser.Block) -> tuple[list[str], list[str]]: while blocklines and not blocklines[-1].strip(): blocklines.pop() + # Separate CHECK_EVAL_BREAKER() macro from end + check_eval_breaker = \ + blocklines != [] and blocklines[-1].strip() == "CHECK_EVAL_BREAKER();" + if check_eval_breaker: + del blocklines[-1] + # Separate PREDICT(...) macros from end predictions: list[str] = [] while blocklines and ( @@ -1119,7 +1128,7 @@ def extract_block_text(block: parser.Block) -> tuple[list[str], list[str]]: predictions.insert(0, m.group(1)) blocklines.pop() - return blocklines, predictions + return blocklines, check_eval_breaker, predictions def always_exits(lines: list[str]) -> bool: diff --git a/Tools/cases_generator/test_generator.py b/Tools/cases_generator/test_generator.py index 9df97d24ab6f43..2c952308b8968d 100644 --- a/Tools/cases_generator/test_generator.py +++ b/Tools/cases_generator/test_generator.py @@ -177,15 +177,16 @@ def test_overlap(): """ run_cases_test(input, output) -def test_predictions(): +def test_predictions_and_eval_breaker(): input = """ inst(OP1, (--)) { } inst(OP2, (--)) { } - inst(OP3, (--)) { + inst(OP3, (arg -- res)) { DEOPT_IF(xxx, OP1); PREDICT(OP2); + CHECK_EVAL_BREAKER(); } """ output = """ @@ -200,8 +201,12 @@ def test_predictions(): } TARGET(OP3) { + PyObject *arg = PEEK(1); + PyObject *res; DEOPT_IF(xxx, OP1); + POKE(1, res); PREDICT(OP2); + CHECK_EVAL_BREAKER(); DISPATCH(); } """ From a6d02274b5e166024bcc283b503d814862be9f56 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 3 Feb 2023 15:00:54 -0800 Subject: [PATCH 02/39] Improve DECREF_INPUTS() - Support arrays - Support conditional inputs - Skip 'unused' and 'null' --- Tools/cases_generator/generate_cases.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 1f16aee373041e..d3cf5750c7fbc5 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -380,9 +380,11 @@ def write_body(self, out: Formatter, dedent: int, cache_adjust: int = 0) -> None # Write the body, substituting a goto for ERROR_IF() and other stuff assert dedent <= 0 extra = " " * -dedent + names_to_skip = self.unmoved_names | frozenset({UNUSED, "null"}) for line in self.block_text: if m := re.match(r"(\s*)ERROR_IF\((.+), (\w+)\);\s*(?://.*)?$", line): space, cond, label = m.groups() + space = extra + space # ERROR_IF() must pop the inputs from the stack. # The code block is responsible for DECREF()ing them. # NOTE: If the label doesn't exist, just add it to ceval.c. @@ -401,16 +403,25 @@ def write_body(self, out: Formatter, dedent: int, cache_adjust: int = 0) -> None symbolic = "" if symbolic: out.write_raw( - f"{extra}{space}if ({cond}) {{ STACK_SHRINK({symbolic}); goto {label}; }}\n" + f"{space}if ({cond}) {{ STACK_SHRINK({symbolic}); goto {label}; }}\n" ) else: - out.write_raw(f"{extra}{space}if ({cond}) goto {label};\n") + out.write_raw(f"{space}if ({cond}) goto {label};\n") elif m := re.match(r"(\s*)DECREF_INPUTS\(\);\s*(?://.*)?$", line): if not self.register: - space = m.group(1) + space = extra + m.group(1) for ieff in self.input_effects: - if ieff.name not in self.unmoved_names: - out.write_raw(f"{extra}{space}Py_DECREF({ieff.name});\n") + if ieff.name in names_to_skip: + continue + if ieff.size: + out.write_raw( + f"{space}for (int _i = {ieff.size}; --_i >= 0;) {{\n" + ) + out.write_raw(f"{space} Py_DECREF({ieff.name}[_i]);\n") + out.write_raw(f"{space}}}\n") + else: + decref = "XDECREF" if ieff.cond else "DECREF" + out.write_raw(f"{space}Py_{decref}({ieff.name});\n") else: out.write_raw(extra + line) From bd63af8b5304c49d529ce26a2446f1563305b8f9 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 1 Feb 2023 11:41:16 -0800 Subject: [PATCH 03/39] Preliminaries - Move CALL_BOUND_METHOD_EXACT_ARGS into place - Add a commented-out family definition --- Python/bytecodes.c | 59 ++++++++++++++++++++++++-------------- Python/generated_cases.c.h | 26 ++++++++--------- Python/opcode_metadata.h | 10 +++---- 3 files changed, 55 insertions(+), 40 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8993567ac82206..5844bf0209f774 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2346,26 +2346,35 @@ dummy_func( assert(oparg & 1); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_BOUND_METHOD_EXACT_ARGS) { - DEOPT_IF(is_method(stack_pointer, oparg), CALL); - PyObject *function = PEEK(oparg + 1); - DEOPT_IF(Py_TYPE(function) != &PyMethod_Type, CALL); - STAT_INC(CALL, hit); - PyObject *self = ((PyMethodObject *)function)->im_self; - PEEK(oparg + 1) = Py_NewRef(self); - PyObject *meth = ((PyMethodObject *)function)->im_func; - PEEK(oparg + 2) = Py_NewRef(meth); - Py_DECREF(function); - GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); - } - inst(KW_NAMES, (--)) { assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(consts)); kwnames = GETITEM(consts, oparg); } + // Cache layout: counter/1, func_version/2, min_args/1 + // Neither CALL_INTRINSIC_1 nor CALL_FUNCTION_EX are members! + // family(call, INLINE_CACHE_ENTRIES_CALL) = { + // CALL, + // CALL_BOUND_METHOD_EXACT_ARGS, + // CALL_PY_EXACT_ARGS, + // CALL_PY_WITH_DEFAULTS, + // CALL_NO_KW_TYPE_1, + // CALL_NO_KW_STR_1, + // CALL_NO_KW_TUPLE_1, + // CALL_BUILTIN_CLASS, + // CALL_NO_KW_BUILTIN_O, + // CALL_NO_KW_BUILTIN_FAST, + // CALL_BUILTIN_FAST_WITH_KEYWORDS, + // CALL_NO_KW_LEN, + // CALL_NO_KW_ISINSTANCE, + // CALL_NO_KW_LIST_APPEND, + // CALL_NO_KW_METHOD_DESCRIPTOR_O, + // CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + // CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, + // CALL_NO_KW_METHOD_DESCRIPTOR_FAST, + // }; + // stack effect: (__0, __array[oparg] -- ) inst(CALL) { #if ENABLE_SPECIALIZATION @@ -2448,6 +2457,20 @@ dummy_func( CHECK_EVAL_BREAKER(); } + // stack effect: (__0, __array[oparg] -- ) + inst(CALL_BOUND_METHOD_EXACT_ARGS) { + DEOPT_IF(is_method(stack_pointer, oparg), CALL); + PyObject *function = PEEK(oparg + 1); + DEOPT_IF(Py_TYPE(function) != &PyMethod_Type, CALL); + STAT_INC(CALL, hit); + PyObject *self = ((PyMethodObject *)function)->im_self; + PEEK(oparg + 1) = Py_NewRef(self); + PyObject *meth = ((PyMethodObject *)function)->im_func; + PEEK(oparg + 2) = Py_NewRef(meth); + Py_DECREF(function); + GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); + } + // stack effect: (__0, __array[oparg] -- ) inst(CALL_PY_EXACT_ARGS) { assert(kwnames == NULL); @@ -3165,14 +3188,6 @@ dummy_func( // Future families go below this point // -family(call, INLINE_CACHE_ENTRIES_CALL) = { - CALL, CALL_PY_EXACT_ARGS, - CALL_PY_WITH_DEFAULTS, CALL_BOUND_METHOD_EXACT_ARGS, CALL_BUILTIN_CLASS, - CALL_BUILTIN_FAST_WITH_KEYWORDS, CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, CALL_NO_KW_BUILTIN_FAST, - CALL_NO_KW_BUILTIN_O, CALL_NO_KW_ISINSTANCE, CALL_NO_KW_LEN, - CALL_NO_KW_LIST_APPEND, CALL_NO_KW_METHOD_DESCRIPTOR_FAST, CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, - CALL_NO_KW_METHOD_DESCRIPTOR_O, CALL_NO_KW_STR_1, CALL_NO_KW_TUPLE_1, - CALL_NO_KW_TYPE_1 }; family(for_iter, INLINE_CACHE_ENTRIES_FOR_ITER) = { FOR_ITER, FOR_ITER_LIST, FOR_ITER_RANGE }; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e524bfcb99d470..3c1ac6223dad6b 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2956,19 +2956,6 @@ DISPATCH(); } - TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { - DEOPT_IF(is_method(stack_pointer, oparg), CALL); - PyObject *function = PEEK(oparg + 1); - DEOPT_IF(Py_TYPE(function) != &PyMethod_Type, CALL); - STAT_INC(CALL, hit); - PyObject *self = ((PyMethodObject *)function)->im_self; - PEEK(oparg + 1) = Py_NewRef(self); - PyObject *meth = ((PyMethodObject *)function)->im_func; - PEEK(oparg + 2) = Py_NewRef(meth); - Py_DECREF(function); - GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); - } - TARGET(KW_NAMES) { assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(consts)); @@ -3059,6 +3046,19 @@ DISPATCH(); } + TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { + DEOPT_IF(is_method(stack_pointer, oparg), CALL); + PyObject *function = PEEK(oparg + 1); + DEOPT_IF(Py_TYPE(function) != &PyMethod_Type, CALL); + STAT_INC(CALL, hit); + PyObject *self = ((PyMethodObject *)function)->im_self; + PEEK(oparg + 1) = Py_NewRef(self); + PyObject *meth = ((PyMethodObject *)function)->im_func; + PEEK(oparg + 2) = Py_NewRef(meth); + Py_DECREF(function); + GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); + } + TARGET(CALL_PY_EXACT_ARGS) { PREDICTED(CALL_PY_EXACT_ARGS); assert(kwnames == NULL); diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 857526c35aa5b6..6f6be277a39ad0 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -284,12 +284,12 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case LOAD_ATTR_METHOD_LAZY_DICT: return 1; - case CALL_BOUND_METHOD_EXACT_ARGS: - return -1; case KW_NAMES: return 0; case CALL: return -1; + case CALL_BOUND_METHOD_EXACT_ARGS: + return -1; case CALL_PY_EXACT_ARGS: return -1; case CALL_PY_WITH_DEFAULTS: @@ -630,12 +630,12 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return ((oparg & 1) ? 1 : 0) + 1; case LOAD_ATTR_METHOD_LAZY_DICT: return ((oparg & 1) ? 1 : 0) + 1; - case CALL_BOUND_METHOD_EXACT_ARGS: - return -1; case KW_NAMES: return 0; case CALL: return -1; + case CALL_BOUND_METHOD_EXACT_ARGS: + return -1; case CALL_PY_EXACT_ARGS: return -1; case CALL_PY_WITH_DEFAULTS: @@ -841,9 +841,9 @@ struct opcode_metadata { [LOAD_ATTR_METHOD_WITH_VALUES] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, [LOAD_ATTR_METHOD_NO_DICT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, [LOAD_ATTR_METHOD_LAZY_DICT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [KW_NAMES] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_PY_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_PY_WITH_DEFAULTS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_TYPE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, From b6ec372e9d76726e5233f51120661eb342cbddda Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 1 Feb 2023 11:59:58 -0800 Subject: [PATCH 04/39] Modernize CALL_BOUND_METHOD_EXACT_ARGS --- Python/bytecodes.c | 21 +++++++++++---------- Python/generated_cases.c.h | 17 +++++++++-------- Python/opcode_metadata.h | 6 +++--- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5844bf0209f774..5d8bcfc51f4a77 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2457,17 +2457,18 @@ dummy_func( CHECK_EVAL_BREAKER(); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_BOUND_METHOD_EXACT_ARGS) { - DEOPT_IF(is_method(stack_pointer, oparg), CALL); - PyObject *function = PEEK(oparg + 1); - DEOPT_IF(Py_TYPE(function) != &PyMethod_Type, CALL); + // Start out with [NULL, method, arg1, arg2, ...] + // Transform to [func, self, arg1, arg2, ...] + // Then fall through to CALL_PY_EXACT_ARGS + inst(CALL_BOUND_METHOD_EXACT_ARGS, (unused/1, unused/2, unused/1, thing1, thing2, unused[oparg] -- unused)) { + DEOPT_IF(thing1 != NULL, CALL); + DEOPT_IF(Py_TYPE(thing2) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); - PyObject *self = ((PyMethodObject *)function)->im_self; - PEEK(oparg + 1) = Py_NewRef(self); - PyObject *meth = ((PyMethodObject *)function)->im_func; - PEEK(oparg + 2) = Py_NewRef(meth); - Py_DECREF(function); + PyObject *self = ((PyMethodObject *)thing2)->im_self; + PEEK(oparg + 1) = Py_NewRef(self); // thing2 + PyObject *meth = ((PyMethodObject *)thing2)->im_func; + PEEK(oparg + 2) = Py_NewRef(meth); // thing1 + Py_DECREF(thing2); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 3c1ac6223dad6b..8e2a3006312f32 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3047,15 +3047,16 @@ } TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { - DEOPT_IF(is_method(stack_pointer, oparg), CALL); - PyObject *function = PEEK(oparg + 1); - DEOPT_IF(Py_TYPE(function) != &PyMethod_Type, CALL); + PyObject *thing2 = PEEK(1 + oparg); + PyObject *thing1 = PEEK(2 + oparg); + DEOPT_IF(thing1 != NULL, CALL); + DEOPT_IF(Py_TYPE(thing2) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); - PyObject *self = ((PyMethodObject *)function)->im_self; - PEEK(oparg + 1) = Py_NewRef(self); - PyObject *meth = ((PyMethodObject *)function)->im_func; - PEEK(oparg + 2) = Py_NewRef(meth); - Py_DECREF(function); + PyObject *self = ((PyMethodObject *)thing2)->im_self; + PEEK(oparg + 1) = Py_NewRef(self); // thing2 + PyObject *meth = ((PyMethodObject *)thing2)->im_func; + PEEK(oparg + 2) = Py_NewRef(meth); // thing1 + Py_DECREF(thing2); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 6f6be277a39ad0..3630826e46bb05 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -289,7 +289,7 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { case CALL: return -1; case CALL_BOUND_METHOD_EXACT_ARGS: - return -1; + return oparg + 2; case CALL_PY_EXACT_ARGS: return -1; case CALL_PY_WITH_DEFAULTS: @@ -635,7 +635,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case CALL: return -1; case CALL_BOUND_METHOD_EXACT_ARGS: - return -1; + return 1; case CALL_PY_EXACT_ARGS: return -1; case CALL_PY_WITH_DEFAULTS: @@ -843,7 +843,7 @@ struct opcode_metadata { [LOAD_ATTR_METHOD_LAZY_DICT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, [KW_NAMES] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_PY_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_PY_WITH_DEFAULTS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_TYPE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, From be49e619ad835dbd3cae0f052cba9ccbe9db0608 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 1 Feb 2023 14:29:39 -0800 Subject: [PATCH 05/39] Modernize CALL_PY_EXACT_ARGS --- Python/bytecodes.c | 14 +++++++------- Python/generated_cases.c.h | 14 +++++++++----- Python/opcode_metadata.h | 6 +++--- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5d8bcfc51f4a77..ec2a68680fcbad 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2472,27 +2472,27 @@ dummy_func( GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_PY_EXACT_ARGS) { + inst(CALL_PY_EXACT_ARGS, (unused/1, func_version/2, unused/1, thing1, thing2, unused[oparg] -- unused)) { assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); - _PyCallCache *cache = (_PyCallCache *)next_instr; - int is_meth = is_method(stack_pointer, oparg); + // _PyCallCache *cache = (_PyCallCache *)next_instr; + int is_meth = thing1 != NULL; int argcount = oparg + is_meth; - PyObject *callable = PEEK(argcount + 1); + PyObject *callable = is_meth ? thing1 : thing2; DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != read_u32(cache->func_version), CALL); + DEOPT_IF(func->func_version != func_version, CALL); PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(code->co_argcount != argcount, CALL); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); STAT_INC(CALL, hit); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(argcount); for (int i = 0; i < argcount; i++) { new_frame->localsplus[i] = stack_pointer[i]; } - STACK_SHRINK(2-is_meth); + STACK_SHRINK(2 - is_meth); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 8e2a3006312f32..52fbd90d0f54df 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3062,25 +3062,29 @@ TARGET(CALL_PY_EXACT_ARGS) { PREDICTED(CALL_PY_EXACT_ARGS); + PyObject *thing2 = PEEK(1 + oparg); + PyObject *thing1 = PEEK(2 + oparg); + uint32_t func_version = read_u32(&next_instr[1].cache); assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); - _PyCallCache *cache = (_PyCallCache *)next_instr; - int is_meth = is_method(stack_pointer, oparg); + // _PyCallCache *cache = (_PyCallCache *)next_instr; + int is_meth = thing1 != NULL; int argcount = oparg + is_meth; - PyObject *callable = PEEK(argcount + 1); + PyObject *callable = is_meth ? thing1 : thing2; DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != read_u32(cache->func_version), CALL); + DEOPT_IF(func->func_version != func_version, CALL); PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(code->co_argcount != argcount, CALL); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); STAT_INC(CALL, hit); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(argcount); for (int i = 0; i < argcount; i++) { new_frame->localsplus[i] = stack_pointer[i]; } - STACK_SHRINK(2-is_meth); + STACK_SHRINK(2 - is_meth); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 3630826e46bb05..8930d31b995ba7 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -291,7 +291,7 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { case CALL_BOUND_METHOD_EXACT_ARGS: return oparg + 2; case CALL_PY_EXACT_ARGS: - return -1; + return oparg + 2; case CALL_PY_WITH_DEFAULTS: return -1; case CALL_NO_KW_TYPE_1: @@ -637,7 +637,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case CALL_BOUND_METHOD_EXACT_ARGS: return 1; case CALL_PY_EXACT_ARGS: - return -1; + return 1; case CALL_PY_WITH_DEFAULTS: return -1; case CALL_NO_KW_TYPE_1: @@ -844,7 +844,7 @@ struct opcode_metadata { [KW_NAMES] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_BOUND_METHOD_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_PY_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [CALL_PY_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_PY_WITH_DEFAULTS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_TYPE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_STR_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, From 18cacf514e58a666145b53056f38b40f2c8036e8 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 1 Feb 2023 14:57:46 -0800 Subject: [PATCH 06/39] Modernize CALL (not too happy with it) --- Python/bytecodes.c | 42 +++++++++++++++++++++++--------------- Python/generated_cases.c.h | 27 ++++++++++++++---------- Python/opcode_metadata.h | 6 +++--- 3 files changed, 44 insertions(+), 31 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ec2a68680fcbad..8da593317c5dcf 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2354,10 +2354,10 @@ dummy_func( // Cache layout: counter/1, func_version/2, min_args/1 // Neither CALL_INTRINSIC_1 nor CALL_FUNCTION_EX are members! - // family(call, INLINE_CACHE_ENTRIES_CALL) = { - // CALL, - // CALL_BOUND_METHOD_EXACT_ARGS, - // CALL_PY_EXACT_ARGS, + family(call, INLINE_CACHE_ENTRIES_CALL) = { + CALL, + CALL_BOUND_METHOD_EXACT_ARGS, + CALL_PY_EXACT_ARGS, // CALL_PY_WITH_DEFAULTS, // CALL_NO_KW_TYPE_1, // CALL_NO_KW_STR_1, @@ -2373,15 +2373,20 @@ dummy_func( // CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, // CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, // CALL_NO_KW_METHOD_DESCRIPTOR_FAST, - // }; + }; - // stack effect: (__0, __array[oparg] -- ) - inst(CALL) { + // Stack is either + // [NULL, function, arg1, arg2, ...] + // or + // [method, self, arg1, arg2, ...] + // (Some args may be keywords, see KW_NAMES, which sets 'kwnames'.) + // It will be replaced with [result]. + inst(CALL, (unused/1, unused/2, unused/1, thing1, thing2, unused[oparg] -- unused)) { #if ENABLE_SPECIALIZATION _PyCallCache *cache = (_PyCallCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); - int is_meth = is_method(stack_pointer, oparg); + int is_meth = thing1 != NULL; int nargs = oparg + is_meth; PyObject *callable = PEEK(nargs + 1); next_instr--; @@ -2392,18 +2397,18 @@ dummy_func( DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ int total_args, is_meth; - is_meth = is_method(stack_pointer, oparg); - PyObject *function = PEEK(oparg + 1); + is_meth = thing1 != NULL; + PyObject *function = thing2; if (!is_meth && Py_TYPE(function) == &PyMethod_Type) { PyObject *self = ((PyMethodObject *)function)->im_self; - PEEK(oparg+1) = Py_NewRef(self); + PEEK(oparg+1) = thing2 = Py_NewRef(self); PyObject *meth = ((PyMethodObject *)function)->im_func; - PEEK(oparg+2) = Py_NewRef(meth); + PEEK(oparg+2) = thing1 = Py_NewRef(meth); Py_DECREF(function); is_meth = 1; } total_args = oparg + is_meth; - function = PEEK(total_args + 1); + function = is_meth ? thing1 : thing2; int positional_args = total_args - KWNAMES_LEN(); // Check if the call can be inlined or not if (Py_TYPE(function) == &PyFunction_Type && @@ -2412,13 +2417,14 @@ dummy_func( { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(function))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(function)); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(total_args); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)function, locals, stack_pointer, positional_args, kwnames ); kwnames = NULL; - STACK_SHRINK(2-is_meth); + STACK_SHRINK(2 - is_meth); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { @@ -2431,30 +2437,32 @@ dummy_func( PyObject *res; if (cframe.use_tracing) { res = trace_call_function( - tstate, function, stack_pointer-total_args, + tstate, function, stack_pointer - total_args, positional_args, kwnames); } else { res = PyObject_Vectorcall( - function, stack_pointer-total_args, + function, stack_pointer - total_args, positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); } kwnames = NULL; assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(function); + // Manipulate stack directly since we leave using DISPATCH(). /* Clear the stack */ STACK_SHRINK(total_args); for (int i = 0; i < total_args; i++) { Py_DECREF(stack_pointer[i]); } - STACK_SHRINK(2-is_meth); + STACK_SHRINK(2 - is_meth); PUSH(res); if (res == NULL) { goto error; } JUMPBY(INLINE_CACHE_ENTRIES_CALL); CHECK_EVAL_BREAKER(); + DISPATCH(); // Prevents generator emitting the epologue. } // Start out with [NULL, method, arg1, arg2, ...] diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 52fbd90d0f54df..f902702549e1ab 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2965,11 +2965,14 @@ TARGET(CALL) { PREDICTED(CALL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 4, "incorrect cache size"); + PyObject *thing2 = PEEK(1 + oparg); + PyObject *thing1 = PEEK(2 + oparg); #if ENABLE_SPECIALIZATION _PyCallCache *cache = (_PyCallCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); - int is_meth = is_method(stack_pointer, oparg); + int is_meth = thing1 != NULL; int nargs = oparg + is_meth; PyObject *callable = PEEK(nargs + 1); next_instr--; @@ -2980,18 +2983,18 @@ DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ int total_args, is_meth; - is_meth = is_method(stack_pointer, oparg); - PyObject *function = PEEK(oparg + 1); + is_meth = thing1 != NULL; + PyObject *function = thing2; if (!is_meth && Py_TYPE(function) == &PyMethod_Type) { PyObject *self = ((PyMethodObject *)function)->im_self; - PEEK(oparg+1) = Py_NewRef(self); + PEEK(oparg+1) = thing2 = Py_NewRef(self); PyObject *meth = ((PyMethodObject *)function)->im_func; - PEEK(oparg+2) = Py_NewRef(meth); + PEEK(oparg+2) = thing1 = Py_NewRef(meth); Py_DECREF(function); is_meth = 1; } total_args = oparg + is_meth; - function = PEEK(total_args + 1); + function = is_meth ? thing1 : thing2; int positional_args = total_args - KWNAMES_LEN(); // Check if the call can be inlined or not if (Py_TYPE(function) == &PyFunction_Type && @@ -3000,13 +3003,14 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(function))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(function)); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(total_args); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)function, locals, stack_pointer, positional_args, kwnames ); kwnames = NULL; - STACK_SHRINK(2-is_meth); + STACK_SHRINK(2 - is_meth); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { @@ -3019,31 +3023,32 @@ PyObject *res; if (cframe.use_tracing) { res = trace_call_function( - tstate, function, stack_pointer-total_args, + tstate, function, stack_pointer - total_args, positional_args, kwnames); } else { res = PyObject_Vectorcall( - function, stack_pointer-total_args, + function, stack_pointer - total_args, positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); } kwnames = NULL; assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(function); + // Manipulate stack directly since we leave using DISPATCH(). /* Clear the stack */ STACK_SHRINK(total_args); for (int i = 0; i < total_args; i++) { Py_DECREF(stack_pointer[i]); } - STACK_SHRINK(2-is_meth); + STACK_SHRINK(2 - is_meth); PUSH(res); if (res == NULL) { goto error; } JUMPBY(INLINE_CACHE_ENTRIES_CALL); CHECK_EVAL_BREAKER(); - DISPATCH(); + DISPATCH(); // Prevents generator emitting the epologue. } TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 8930d31b995ba7..47afa5ccc000ec 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -287,7 +287,7 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { case KW_NAMES: return 0; case CALL: - return -1; + return oparg + 2; case CALL_BOUND_METHOD_EXACT_ARGS: return oparg + 2; case CALL_PY_EXACT_ARGS: @@ -633,7 +633,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case KW_NAMES: return 0; case CALL: - return -1; + return 1; case CALL_BOUND_METHOD_EXACT_ARGS: return 1; case CALL_PY_EXACT_ARGS: @@ -842,7 +842,7 @@ struct opcode_metadata { [LOAD_ATTR_METHOD_NO_DICT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, [LOAD_ATTR_METHOD_LAZY_DICT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, [KW_NAMES] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [CALL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [CALL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_BOUND_METHOD_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_PY_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_PY_WITH_DEFAULTS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, From d69ca8e6d6b92e9fece0e4893f594d0e70eb9fce Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 1 Feb 2023 15:18:38 -0800 Subject: [PATCH 07/39] Modernize CALL_PY_WITH_DEFAULTS --- Python/bytecodes.c | 21 +++++++++------------ Python/generated_cases.c.h | 20 +++++++++++--------- Python/opcode_metadata.h | 6 +++--- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8da593317c5dcf..7e975c52e553e1 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2358,7 +2358,7 @@ dummy_func( CALL, CALL_BOUND_METHOD_EXACT_ARGS, CALL_PY_EXACT_ARGS, - // CALL_PY_WITH_DEFAULTS, + CALL_PY_WITH_DEFAULTS, // CALL_NO_KW_TYPE_1, // CALL_NO_KW_STR_1, // CALL_NO_KW_TUPLE_1, @@ -2505,34 +2505,31 @@ dummy_func( DISPATCH_INLINED(new_frame); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_PY_WITH_DEFAULTS) { + inst(CALL_PY_WITH_DEFAULTS, (unused/1, func_version/2, min_args/1, thing1, thing2, unused[oparg] -- unused)) { assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); - _PyCallCache *cache = (_PyCallCache *)next_instr; - int is_meth = is_method(stack_pointer, oparg); + int is_meth = thing1 != NULL; int argcount = oparg + is_meth; - PyObject *callable = PEEK(argcount + 1); + PyObject *callable = is_meth ? thing1 : thing2; DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != read_u32(cache->func_version), CALL); + DEOPT_IF(func->func_version != func_version, CALL); PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(argcount > code->co_argcount, CALL); - int minargs = cache->min_args; - DEOPT_IF(argcount < minargs, CALL); + DEOPT_IF(argcount < min_args, CALL); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); STAT_INC(CALL, hit); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(argcount); for (int i = 0; i < argcount; i++) { new_frame->localsplus[i] = stack_pointer[i]; } for (int i = argcount; i < code->co_argcount; i++) { - PyObject *def = PyTuple_GET_ITEM(func->func_defaults, - i - minargs); + PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args); new_frame->localsplus[i] = Py_NewRef(def); } - STACK_SHRINK(2-is_meth); + STACK_SHRINK(2 - is_meth); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index f902702549e1ab..75c26d11bb5a4b 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3095,32 +3095,34 @@ } TARGET(CALL_PY_WITH_DEFAULTS) { + PyObject *thing2 = PEEK(1 + oparg); + PyObject *thing1 = PEEK(2 + oparg); + uint32_t func_version = read_u32(&next_instr[1].cache); + uint16_t min_args = read_u16(&next_instr[3].cache); assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); - _PyCallCache *cache = (_PyCallCache *)next_instr; - int is_meth = is_method(stack_pointer, oparg); + int is_meth = thing1 != NULL; int argcount = oparg + is_meth; - PyObject *callable = PEEK(argcount + 1); + PyObject *callable = is_meth ? thing1 : thing2; DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != read_u32(cache->func_version), CALL); + DEOPT_IF(func->func_version != func_version, CALL); PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(argcount > code->co_argcount, CALL); - int minargs = cache->min_args; - DEOPT_IF(argcount < minargs, CALL); + DEOPT_IF(argcount < min_args, CALL); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); STAT_INC(CALL, hit); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(argcount); for (int i = 0; i < argcount; i++) { new_frame->localsplus[i] = stack_pointer[i]; } for (int i = argcount; i < code->co_argcount; i++) { - PyObject *def = PyTuple_GET_ITEM(func->func_defaults, - i - minargs); + PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args); new_frame->localsplus[i] = Py_NewRef(def); } - STACK_SHRINK(2-is_meth); + STACK_SHRINK(2 - is_meth); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 47afa5ccc000ec..f1fd4e56406b76 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -293,7 +293,7 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { case CALL_PY_EXACT_ARGS: return oparg + 2; case CALL_PY_WITH_DEFAULTS: - return -1; + return oparg + 2; case CALL_NO_KW_TYPE_1: return -1; case CALL_NO_KW_STR_1: @@ -639,7 +639,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case CALL_PY_EXACT_ARGS: return 1; case CALL_PY_WITH_DEFAULTS: - return -1; + return 1; case CALL_NO_KW_TYPE_1: return -1; case CALL_NO_KW_STR_1: @@ -845,7 +845,7 @@ struct opcode_metadata { [CALL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_BOUND_METHOD_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_PY_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_PY_WITH_DEFAULTS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [CALL_PY_WITH_DEFAULTS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_TYPE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_STR_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_TUPLE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, From 09701cf4d75748399aa077d573bba19a9ea6b3e7 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 3 Feb 2023 10:29:16 -0800 Subject: [PATCH 08/39] Modernize CALL_NO_KW_TYPE_1 --- Python/bytecodes.c | 17 ++++++----------- Python/generated_cases.c.h | 20 ++++++++++++-------- Python/opcode_metadata.h | 6 +++--- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 7e975c52e553e1..320a1e88e19e0e 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2359,7 +2359,7 @@ dummy_func( CALL_BOUND_METHOD_EXACT_ARGS, CALL_PY_EXACT_ARGS, CALL_PY_WITH_DEFAULTS, - // CALL_NO_KW_TYPE_1, + CALL_NO_KW_TYPE_1, // CALL_NO_KW_STR_1, // CALL_NO_KW_TUPLE_1, // CALL_BUILTIN_CLASS, @@ -2534,22 +2534,17 @@ dummy_func( DISPATCH_INLINED(new_frame); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_NO_KW_TYPE_1) { + inst(CALL_NO_KW_TYPE_1, (unused/1, unused/2, unused/1, null, callable, args[oparg] -- res)) { assert(kwnames == NULL); assert(cframe.use_tracing == 0); assert(oparg == 1); - DEOPT_IF(is_method(stack_pointer, 1), CALL); - PyObject *obj = TOP(); - PyObject *callable = SECOND(); + DEOPT_IF(null != NULL, CALL); + PyObject *obj = args[0]; DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); STAT_INC(CALL, hit); - JUMPBY(INLINE_CACHE_ENTRIES_CALL); - PyObject *res = Py_NewRef(Py_TYPE(obj)); - Py_DECREF(callable); + res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); - STACK_SHRINK(2); - SET_TOP(res); + Py_DECREF(callable); } // stack effect: (__0, __array[oparg] -- ) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 75c26d11bb5a4b..176c5508309c05 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3128,20 +3128,24 @@ } TARGET(CALL_NO_KW_TYPE_1) { + PyObject **args = &PEEK(oparg); + PyObject *callable = PEEK(1 + oparg); + PyObject *null = PEEK(2 + oparg); + PyObject *res; assert(kwnames == NULL); assert(cframe.use_tracing == 0); assert(oparg == 1); - DEOPT_IF(is_method(stack_pointer, 1), CALL); - PyObject *obj = TOP(); - PyObject *callable = SECOND(); + DEOPT_IF(null != NULL, CALL); + PyObject *obj = args[0]; DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); STAT_INC(CALL, hit); - JUMPBY(INLINE_CACHE_ENTRIES_CALL); - PyObject *res = Py_NewRef(Py_TYPE(obj)); - Py_DECREF(callable); + res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); - STACK_SHRINK(2); - SET_TOP(res); + Py_DECREF(callable); + STACK_SHRINK(oparg); + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(4); DISPATCH(); } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index f1fd4e56406b76..02dfdd32b69607 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -295,7 +295,7 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { case CALL_PY_WITH_DEFAULTS: return oparg + 2; case CALL_NO_KW_TYPE_1: - return -1; + return oparg + 2; case CALL_NO_KW_STR_1: return -1; case CALL_NO_KW_TUPLE_1: @@ -641,7 +641,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case CALL_PY_WITH_DEFAULTS: return 1; case CALL_NO_KW_TYPE_1: - return -1; + return 1; case CALL_NO_KW_STR_1: return -1; case CALL_NO_KW_TUPLE_1: @@ -846,7 +846,7 @@ struct opcode_metadata { [CALL_BOUND_METHOD_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_PY_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_PY_WITH_DEFAULTS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_TYPE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [CALL_NO_KW_TYPE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_STR_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_TUPLE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_BUILTIN_CLASS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, From 389601de960b81bc8dea31ebdfc0c16ad43d122b Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 3 Feb 2023 10:57:01 -0800 Subject: [PATCH 09/39] Modernize CALL_NO_KW_STR_1 --- Python/bytecodes.c | 21 +++++++-------------- Python/generated_cases.c.h | 24 +++++++++++++----------- Python/opcode_metadata.h | 6 +++--- 3 files changed, 23 insertions(+), 28 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 320a1e88e19e0e..471a461cee83a9 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2360,7 +2360,7 @@ dummy_func( CALL_PY_EXACT_ARGS, CALL_PY_WITH_DEFAULTS, CALL_NO_KW_TYPE_1, - // CALL_NO_KW_STR_1, + CALL_NO_KW_STR_1, // CALL_NO_KW_TUPLE_1, // CALL_BUILTIN_CLASS, // CALL_NO_KW_BUILTIN_O, @@ -2547,25 +2547,18 @@ dummy_func( Py_DECREF(callable); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_NO_KW_STR_1) { + inst(CALL_NO_KW_STR_1, (unused/1, unused/2, unused/1, null, callable, args[oparg] -- res)) { assert(kwnames == NULL); assert(cframe.use_tracing == 0); assert(oparg == 1); - DEOPT_IF(is_method(stack_pointer, 1), CALL); - PyObject *callable = PEEK(2); + DEOPT_IF(null != NULL, CALL); DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); STAT_INC(CALL, hit); - PyObject *arg = TOP(); - PyObject *res = PyObject_Str(arg); + PyObject *arg = args[0]; + res = PyObject_Str(arg); Py_DECREF(arg); - Py_DECREF(&PyUnicode_Type); - STACK_SHRINK(2); - SET_TOP(res); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + Py_DECREF(&PyUnicode_Type); // I.e., callable + ERROR_IF(res == NULL, error); CHECK_EVAL_BREAKER(); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 176c5508309c05..08674452336a49 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3150,23 +3150,25 @@ } TARGET(CALL_NO_KW_STR_1) { + PyObject **args = &PEEK(oparg); + PyObject *callable = PEEK(1 + oparg); + PyObject *null = PEEK(2 + oparg); + PyObject *res; assert(kwnames == NULL); assert(cframe.use_tracing == 0); assert(oparg == 1); - DEOPT_IF(is_method(stack_pointer, 1), CALL); - PyObject *callable = PEEK(2); + DEOPT_IF(null != NULL, CALL); DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); STAT_INC(CALL, hit); - PyObject *arg = TOP(); - PyObject *res = PyObject_Str(arg); + PyObject *arg = args[0]; + res = PyObject_Str(arg); Py_DECREF(arg); - Py_DECREF(&PyUnicode_Type); - STACK_SHRINK(2); - SET_TOP(res); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + Py_DECREF(&PyUnicode_Type); // I.e., callable + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(4); CHECK_EVAL_BREAKER(); DISPATCH(); } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 02dfdd32b69607..be986cac79b811 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -297,7 +297,7 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { case CALL_NO_KW_TYPE_1: return oparg + 2; case CALL_NO_KW_STR_1: - return -1; + return oparg + 2; case CALL_NO_KW_TUPLE_1: return -1; case CALL_BUILTIN_CLASS: @@ -643,7 +643,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case CALL_NO_KW_TYPE_1: return 1; case CALL_NO_KW_STR_1: - return -1; + return 1; case CALL_NO_KW_TUPLE_1: return -1; case CALL_BUILTIN_CLASS: @@ -847,7 +847,7 @@ struct opcode_metadata { [CALL_PY_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_PY_WITH_DEFAULTS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_TYPE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_STR_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [CALL_NO_KW_STR_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_TUPLE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_BUILTIN_CLASS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_BUILTIN_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, From 9a7b75a9a53167cad266d82b3174dca9ebf02c80 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 3 Feb 2023 16:59:06 -0800 Subject: [PATCH 10/39] Modernize CALL_NO_KW_TUPLE_1 --- Python/bytecodes.c | 21 +++++++-------------- Python/generated_cases.c.h | 24 +++++++++++++----------- Python/opcode_metadata.h | 6 +++--- 3 files changed, 23 insertions(+), 28 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 471a461cee83a9..4ac154e7767784 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2361,7 +2361,7 @@ dummy_func( CALL_PY_WITH_DEFAULTS, CALL_NO_KW_TYPE_1, CALL_NO_KW_STR_1, - // CALL_NO_KW_TUPLE_1, + CALL_NO_KW_TUPLE_1, // CALL_BUILTIN_CLASS, // CALL_NO_KW_BUILTIN_O, // CALL_NO_KW_BUILTIN_FAST, @@ -2562,24 +2562,17 @@ dummy_func( CHECK_EVAL_BREAKER(); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_NO_KW_TUPLE_1) { + inst(CALL_NO_KW_TUPLE_1, (unused/1, unused/2, unused/1, null, callable, args[oparg] -- res)) { assert(kwnames == NULL); assert(oparg == 1); - DEOPT_IF(is_method(stack_pointer, 1), CALL); - PyObject *callable = PEEK(2); + DEOPT_IF(null != NULL, CALL); DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); STAT_INC(CALL, hit); - PyObject *arg = TOP(); - PyObject *res = PySequence_Tuple(arg); + PyObject *arg = args[0]; + res = PySequence_Tuple(arg); Py_DECREF(arg); - Py_DECREF(&PyTuple_Type); - STACK_SHRINK(2); - SET_TOP(res); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + Py_DECREF(&PyTuple_Type); // I.e., tuple + ERROR_IF(res == NULL, error); CHECK_EVAL_BREAKER(); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 08674452336a49..81cd5f669137fa 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3174,22 +3174,24 @@ } TARGET(CALL_NO_KW_TUPLE_1) { + PyObject **args = &PEEK(oparg); + PyObject *callable = PEEK(1 + oparg); + PyObject *null = PEEK(2 + oparg); + PyObject *res; assert(kwnames == NULL); assert(oparg == 1); - DEOPT_IF(is_method(stack_pointer, 1), CALL); - PyObject *callable = PEEK(2); + DEOPT_IF(null != NULL, CALL); DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); STAT_INC(CALL, hit); - PyObject *arg = TOP(); - PyObject *res = PySequence_Tuple(arg); + PyObject *arg = args[0]; + res = PySequence_Tuple(arg); Py_DECREF(arg); - Py_DECREF(&PyTuple_Type); - STACK_SHRINK(2); - SET_TOP(res); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + Py_DECREF(&PyTuple_Type); // I.e., tuple + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(4); CHECK_EVAL_BREAKER(); DISPATCH(); } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index be986cac79b811..dbe931c22758fe 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -299,7 +299,7 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { case CALL_NO_KW_STR_1: return oparg + 2; case CALL_NO_KW_TUPLE_1: - return -1; + return oparg + 2; case CALL_BUILTIN_CLASS: return -1; case CALL_NO_KW_BUILTIN_O: @@ -645,7 +645,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case CALL_NO_KW_STR_1: return 1; case CALL_NO_KW_TUPLE_1: - return -1; + return 1; case CALL_BUILTIN_CLASS: return -1; case CALL_NO_KW_BUILTIN_O: @@ -848,7 +848,7 @@ struct opcode_metadata { [CALL_PY_WITH_DEFAULTS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_TYPE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_STR_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_TUPLE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [CALL_NO_KW_TUPLE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_BUILTIN_CLASS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_BUILTIN_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_BUILTIN_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, From af19fa6718861463c3b8ffb10a4edc86e93a1807 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 3 Feb 2023 17:12:20 -0800 Subject: [PATCH 11/39] Modernize CALL_BUILTIN_CLASS (a different way) --- Python/bytecodes.c | 29 +++++++++++++---------------- Python/generated_cases.c.h | 32 +++++++++++++++++++------------- Python/opcode_metadata.h | 6 +++--- 3 files changed, 35 insertions(+), 32 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 4ac154e7767784..c1d67ebf39d7c4 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2362,7 +2362,7 @@ dummy_func( CALL_NO_KW_TYPE_1, CALL_NO_KW_STR_1, CALL_NO_KW_TUPLE_1, - // CALL_BUILTIN_CLASS, + CALL_BUILTIN_CLASS, // CALL_NO_KW_BUILTIN_O, // CALL_NO_KW_BUILTIN_FAST, // CALL_BUILTIN_FAST_WITH_KEYWORDS, @@ -2576,31 +2576,28 @@ dummy_func( CHECK_EVAL_BREAKER(); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_BUILTIN_CLASS) { - int is_meth = is_method(stack_pointer, oparg); - int total_args = oparg + is_meth; + inst(CALL_BUILTIN_CLASS, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) { + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + callable = method; + args--; + total_args++; + } int kwnames_len = KWNAMES_LEN(); - PyObject *callable = PEEK(total_args + 1); DEOPT_IF(!PyType_Check(callable), CALL); PyTypeObject *tp = (PyTypeObject *)callable; DEOPT_IF(tp->tp_vectorcall == NULL, CALL); STAT_INC(CALL, hit); - STACK_SHRINK(total_args); - PyObject *res = tp->tp_vectorcall((PyObject *)tp, stack_pointer, - total_args-kwnames_len, kwnames); + res = tp->tp_vectorcall((PyObject *)tp, args, + total_args - kwnames_len, kwnames); kwnames = NULL; /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(stack_pointer[i]); + Py_DECREF(args[i]); } Py_DECREF(tp); - STACK_SHRINK(1-is_meth); - SET_TOP(res); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + ERROR_IF(res == NULL, error); CHECK_EVAL_BREAKER(); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 81cd5f669137fa..c21873c9e14ba4 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3197,29 +3197,35 @@ } TARGET(CALL_BUILTIN_CLASS) { - int is_meth = is_method(stack_pointer, oparg); - int total_args = oparg + is_meth; + PyObject **args = &PEEK(oparg); + PyObject *callable = PEEK(1 + oparg); + PyObject *method = PEEK(2 + oparg); + PyObject *res; + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + callable = method; + args--; + total_args++; + } int kwnames_len = KWNAMES_LEN(); - PyObject *callable = PEEK(total_args + 1); DEOPT_IF(!PyType_Check(callable), CALL); PyTypeObject *tp = (PyTypeObject *)callable; DEOPT_IF(tp->tp_vectorcall == NULL, CALL); STAT_INC(CALL, hit); - STACK_SHRINK(total_args); - PyObject *res = tp->tp_vectorcall((PyObject *)tp, stack_pointer, - total_args-kwnames_len, kwnames); + res = tp->tp_vectorcall((PyObject *)tp, args, + total_args - kwnames_len, kwnames); kwnames = NULL; /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(stack_pointer[i]); + Py_DECREF(args[i]); } Py_DECREF(tp); - STACK_SHRINK(1-is_meth); - SET_TOP(res); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(4); CHECK_EVAL_BREAKER(); DISPATCH(); } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index dbe931c22758fe..27ee730b3020d6 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -301,7 +301,7 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { case CALL_NO_KW_TUPLE_1: return oparg + 2; case CALL_BUILTIN_CLASS: - return -1; + return oparg + 2; case CALL_NO_KW_BUILTIN_O: return -1; case CALL_NO_KW_BUILTIN_FAST: @@ -647,7 +647,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case CALL_NO_KW_TUPLE_1: return 1; case CALL_BUILTIN_CLASS: - return -1; + return 1; case CALL_NO_KW_BUILTIN_O: return -1; case CALL_NO_KW_BUILTIN_FAST: @@ -849,7 +849,7 @@ struct opcode_metadata { [CALL_NO_KW_TYPE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_STR_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_TUPLE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_BUILTIN_CLASS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [CALL_BUILTIN_CLASS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_BUILTIN_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_BUILTIN_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, From 05e4ea8b5753f9eb0486b8101caf2ec386f68823 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sat, 4 Feb 2023 11:29:31 -0800 Subject: [PATCH 12/39] Modernize CALL_NO_KW_BUILTIN_O --- Python/bytecodes.c | 22 +++++++++------------- Python/generated_cases.c.h | 25 +++++++++++++++---------- Python/opcode_metadata.h | 6 +++--- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c1d67ebf39d7c4..f2727280d8c779 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2363,7 +2363,7 @@ dummy_func( CALL_NO_KW_STR_1, CALL_NO_KW_TUPLE_1, CALL_BUILTIN_CLASS, - // CALL_NO_KW_BUILTIN_O, + CALL_NO_KW_BUILTIN_O, // CALL_NO_KW_BUILTIN_FAST, // CALL_BUILTIN_FAST_WITH_KEYWORDS, // CALL_NO_KW_LEN, @@ -2601,15 +2601,16 @@ dummy_func( CHECK_EVAL_BREAKER(); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_NO_KW_BUILTIN_O) { + inst(CALL_NO_KW_BUILTIN_O, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) { assert(cframe.use_tracing == 0); /* Builtin METH_O functions */ assert(kwnames == NULL); - int is_meth = is_method(stack_pointer, oparg); + int is_meth = method != NULL; int total_args = oparg + is_meth; DEOPT_IF(total_args != 1, CALL); - PyObject *callable = PEEK(total_args + 1); + if (is_meth) { + callable = method; + } DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); STAT_INC(CALL, hit); @@ -2619,19 +2620,14 @@ dummy_func( if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { goto error; } - PyObject *arg = TOP(); - PyObject *res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); + PyObject *arg = args[-is_meth]; + res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(arg); Py_DECREF(callable); - STACK_SHRINK(2-is_meth); - SET_TOP(res); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + ERROR_IF(res == NULL, error); CHECK_EVAL_BREAKER(); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index c21873c9e14ba4..393f56025b11b4 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3231,13 +3231,19 @@ } TARGET(CALL_NO_KW_BUILTIN_O) { + PyObject **args = &PEEK(oparg); + PyObject *callable = PEEK(1 + oparg); + PyObject *method = PEEK(2 + oparg); + PyObject *res; assert(cframe.use_tracing == 0); /* Builtin METH_O functions */ assert(kwnames == NULL); - int is_meth = is_method(stack_pointer, oparg); + int is_meth = method != NULL; int total_args = oparg + is_meth; DEOPT_IF(total_args != 1, CALL); - PyObject *callable = PEEK(total_args + 1); + if (is_meth) { + callable = method; + } DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); STAT_INC(CALL, hit); @@ -3247,19 +3253,18 @@ if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { goto error; } - PyObject *arg = TOP(); - PyObject *res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); + PyObject *arg = args[-is_meth]; + res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(arg); Py_DECREF(callable); - STACK_SHRINK(2-is_meth); - SET_TOP(res); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(4); CHECK_EVAL_BREAKER(); DISPATCH(); } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 27ee730b3020d6..fc08116ff4e41f 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -303,7 +303,7 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { case CALL_BUILTIN_CLASS: return oparg + 2; case CALL_NO_KW_BUILTIN_O: - return -1; + return oparg + 2; case CALL_NO_KW_BUILTIN_FAST: return -1; case CALL_BUILTIN_FAST_WITH_KEYWORDS: @@ -649,7 +649,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case CALL_BUILTIN_CLASS: return 1; case CALL_NO_KW_BUILTIN_O: - return -1; + return 1; case CALL_NO_KW_BUILTIN_FAST: return -1; case CALL_BUILTIN_FAST_WITH_KEYWORDS: @@ -850,7 +850,7 @@ struct opcode_metadata { [CALL_NO_KW_STR_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_TUPLE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_BUILTIN_CLASS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_BUILTIN_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [CALL_NO_KW_BUILTIN_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_BUILTIN_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_LEN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, From 491de2841f23b18bfeb67cd4e51b8c1807cb079e Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sat, 4 Feb 2023 11:36:38 -0800 Subject: [PATCH 13/39] Modernize CALL_NO_KW_BUILTIN_FAST --- Python/bytecodes.c | 28 +++++++++++----------------- Python/generated_cases.c.h | 31 +++++++++++++++++-------------- Python/opcode_metadata.h | 6 +++--- 3 files changed, 31 insertions(+), 34 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index f2727280d8c779..fecd44b32a1eb2 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2364,7 +2364,7 @@ dummy_func( CALL_NO_KW_TUPLE_1, CALL_BUILTIN_CLASS, CALL_NO_KW_BUILTIN_O, - // CALL_NO_KW_BUILTIN_FAST, + CALL_NO_KW_BUILTIN_FAST, // CALL_BUILTIN_FAST_WITH_KEYWORDS, // CALL_NO_KW_LEN, // CALL_NO_KW_ISINSTANCE, @@ -2631,43 +2631,37 @@ dummy_func( CHECK_EVAL_BREAKER(); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_NO_KW_BUILTIN_FAST) { + inst(CALL_NO_KW_BUILTIN_FAST, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) { assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); - int is_meth = is_method(stack_pointer, oparg); + int is_meth = method != NULL; int total_args = oparg + is_meth; - PyObject *callable = PEEK(total_args + 1); + if (is_meth) { + callable = method; + } DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, - CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - STACK_SHRINK(total_args); /* res = func(self, args, nargs) */ - PyObject *res = ((_PyCFunctionFast)(void(*)(void))cfunc)( + res = ((_PyCFunctionFast)(void(*)(void))cfunc)( PyCFunction_GET_SELF(callable), - stack_pointer, + args - is_meth, total_args); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(stack_pointer[i]); + Py_DECREF(args[i - is_meth]); } - STACK_SHRINK(2-is_meth); - PUSH(res); Py_DECREF(callable); - if (res == NULL) { + ERROR_IF(res == NULL, error); /* Not deopting because this doesn't mean our optimization was wrong. `res` can be NULL for valid reasons. Eg. getattr(x, 'invalid'). In those cases an exception is set, so we must handle it. */ - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); CHECK_EVAL_BREAKER(); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 393f56025b11b4..99950e8f3b49d6 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3270,41 +3270,44 @@ } TARGET(CALL_NO_KW_BUILTIN_FAST) { + PyObject **args = &PEEK(oparg); + PyObject *callable = PEEK(1 + oparg); + PyObject *method = PEEK(2 + oparg); + PyObject *res; assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); - int is_meth = is_method(stack_pointer, oparg); + int is_meth = method != NULL; int total_args = oparg + is_meth; - PyObject *callable = PEEK(total_args + 1); + if (is_meth) { + callable = method; + } DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, - CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - STACK_SHRINK(total_args); /* res = func(self, args, nargs) */ - PyObject *res = ((_PyCFunctionFast)(void(*)(void))cfunc)( + res = ((_PyCFunctionFast)(void(*)(void))cfunc)( PyCFunction_GET_SELF(callable), - stack_pointer, + args - is_meth, total_args); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(stack_pointer[i]); + Py_DECREF(args[i - is_meth]); } - STACK_SHRINK(2-is_meth); - PUSH(res); Py_DECREF(callable); - if (res == NULL) { + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } /* Not deopting because this doesn't mean our optimization was wrong. `res` can be NULL for valid reasons. Eg. getattr(x, 'invalid'). In those cases an exception is set, so we must handle it. */ - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + STACK_SHRINK(oparg); + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(4); CHECK_EVAL_BREAKER(); DISPATCH(); } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index fc08116ff4e41f..067be25d682fce 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -305,7 +305,7 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { case CALL_NO_KW_BUILTIN_O: return oparg + 2; case CALL_NO_KW_BUILTIN_FAST: - return -1; + return oparg + 2; case CALL_BUILTIN_FAST_WITH_KEYWORDS: return -1; case CALL_NO_KW_LEN: @@ -651,7 +651,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case CALL_NO_KW_BUILTIN_O: return 1; case CALL_NO_KW_BUILTIN_FAST: - return -1; + return 1; case CALL_BUILTIN_FAST_WITH_KEYWORDS: return -1; case CALL_NO_KW_LEN: @@ -851,7 +851,7 @@ struct opcode_metadata { [CALL_NO_KW_TUPLE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_BUILTIN_CLASS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_BUILTIN_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_BUILTIN_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [CALL_NO_KW_BUILTIN_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_LEN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_ISINSTANCE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, From 25bc08ad271f0b9ba90af907ea3d89f2922a9cd2 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sat, 4 Feb 2023 14:24:23 -0800 Subject: [PATCH 14/39] Modernize CALL_BUILTIN_FAST_WITH_KEYWORDS --- Python/bytecodes.c | 25 ++++++++++--------------- Python/generated_cases.c.h | 28 ++++++++++++++++------------ Python/opcode_metadata.h | 6 +++--- 3 files changed, 29 insertions(+), 30 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index fecd44b32a1eb2..ba4a3d76965765 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2365,7 +2365,7 @@ dummy_func( CALL_BUILTIN_CLASS, CALL_NO_KW_BUILTIN_O, CALL_NO_KW_BUILTIN_FAST, - // CALL_BUILTIN_FAST_WITH_KEYWORDS, + CALL_BUILTIN_FAST_WITH_KEYWORDS, // CALL_NO_KW_LEN, // CALL_NO_KW_ISINSTANCE, // CALL_NO_KW_LIST_APPEND, @@ -2665,25 +2665,25 @@ dummy_func( CHECK_EVAL_BREAKER(); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_BUILTIN_FAST_WITH_KEYWORDS) { + inst(CALL_BUILTIN_FAST_WITH_KEYWORDS, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) { assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - int is_meth = is_method(stack_pointer, oparg); + int is_meth = method != NULL; int total_args = oparg + is_meth; - PyObject *callable = PEEK(total_args + 1); + if (is_meth) { + callable = method; + } DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); STAT_INC(CALL, hit); - STACK_SHRINK(total_args); /* res = func(self, args, nargs, kwnames) */ _PyCFunctionFastWithKeywords cfunc = (_PyCFunctionFastWithKeywords)(void(*)(void)) PyCFunction_GET_FUNCTION(callable); - PyObject *res = cfunc( + res = cfunc( PyCFunction_GET_SELF(callable), - stack_pointer, + args - is_meth, total_args - KWNAMES_LEN(), kwnames ); @@ -2692,15 +2692,10 @@ dummy_func( /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(stack_pointer[i]); + Py_DECREF(args[i - is_meth]); } - STACK_SHRINK(2-is_meth); - PUSH(res); Py_DECREF(callable); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + ERROR_IF(res == NULL, error); CHECK_EVAL_BREAKER(); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 99950e8f3b49d6..2cb324d2286783 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3313,23 +3313,28 @@ } TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { + PyObject **args = &PEEK(oparg); + PyObject *callable = PEEK(1 + oparg); + PyObject *method = PEEK(2 + oparg); + PyObject *res; assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - int is_meth = is_method(stack_pointer, oparg); + int is_meth = method != NULL; int total_args = oparg + is_meth; - PyObject *callable = PEEK(total_args + 1); + if (is_meth) { + callable = method; + } DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); STAT_INC(CALL, hit); - STACK_SHRINK(total_args); /* res = func(self, args, nargs, kwnames) */ _PyCFunctionFastWithKeywords cfunc = (_PyCFunctionFastWithKeywords)(void(*)(void)) PyCFunction_GET_FUNCTION(callable); - PyObject *res = cfunc( + res = cfunc( PyCFunction_GET_SELF(callable), - stack_pointer, + args - is_meth, total_args - KWNAMES_LEN(), kwnames ); @@ -3338,15 +3343,14 @@ /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(stack_pointer[i]); + Py_DECREF(args[i - is_meth]); } - STACK_SHRINK(2-is_meth); - PUSH(res); Py_DECREF(callable); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(4); CHECK_EVAL_BREAKER(); DISPATCH(); } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 067be25d682fce..e438c991fc0194 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -307,7 +307,7 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { case CALL_NO_KW_BUILTIN_FAST: return oparg + 2; case CALL_BUILTIN_FAST_WITH_KEYWORDS: - return -1; + return oparg + 2; case CALL_NO_KW_LEN: return -1; case CALL_NO_KW_ISINSTANCE: @@ -653,7 +653,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case CALL_NO_KW_BUILTIN_FAST: return 1; case CALL_BUILTIN_FAST_WITH_KEYWORDS: - return -1; + return 1; case CALL_NO_KW_LEN: return -1; case CALL_NO_KW_ISINSTANCE: @@ -852,7 +852,7 @@ struct opcode_metadata { [CALL_BUILTIN_CLASS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_BUILTIN_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_BUILTIN_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_LEN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_ISINSTANCE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_LIST_APPEND] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, From be8ec59cf91ca2ac77991143981f05e3f4a8cb91 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sat, 4 Feb 2023 14:29:23 -0800 Subject: [PATCH 15/39] Modernize CALL_NO_KW_LEN --- Python/bytecodes.c | 22 +++++++++------------- Python/generated_cases.c.h | 25 +++++++++++++++---------- Python/opcode_metadata.h | 6 +++--- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ba4a3d76965765..1fc926eb74497f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2366,7 +2366,7 @@ dummy_func( CALL_NO_KW_BUILTIN_O, CALL_NO_KW_BUILTIN_FAST, CALL_BUILTIN_FAST_WITH_KEYWORDS, - // CALL_NO_KW_LEN, + CALL_NO_KW_LEN, // CALL_NO_KW_ISINSTANCE, // CALL_NO_KW_LIST_APPEND, // CALL_NO_KW_METHOD_DESCRIPTOR_O, @@ -2699,34 +2699,30 @@ dummy_func( CHECK_EVAL_BREAKER(); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_NO_KW_LEN) { + inst(CALL_NO_KW_LEN, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) { assert(cframe.use_tracing == 0); assert(kwnames == NULL); /* len(o) */ - int is_meth = is_method(stack_pointer, oparg); + int is_meth = method != NULL; int total_args = oparg + is_meth; DEOPT_IF(total_args != 1, CALL); - PyObject *callable = PEEK(total_args + 1); + if (is_meth) { + callable = method; + } PyInterpreterState *interp = _PyInterpreterState_GET(); DEOPT_IF(callable != interp->callable_cache.len, CALL); STAT_INC(CALL, hit); - PyObject *arg = TOP(); + PyObject *arg = args[-is_meth]; Py_ssize_t len_i = PyObject_Length(arg); if (len_i < 0) { goto error; } - PyObject *res = PyLong_FromSsize_t(len_i); + res = PyLong_FromSsize_t(len_i); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - STACK_SHRINK(2-is_meth); - SET_TOP(res); Py_DECREF(callable); Py_DECREF(arg); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + ERROR_IF(res == NULL, error); } // stack effect: (__0, __array[oparg] -- ) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2cb324d2286783..6a3ac6d7c3cdf4 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3356,32 +3356,37 @@ } TARGET(CALL_NO_KW_LEN) { + PyObject **args = &PEEK(oparg); + PyObject *callable = PEEK(1 + oparg); + PyObject *method = PEEK(2 + oparg); + PyObject *res; assert(cframe.use_tracing == 0); assert(kwnames == NULL); /* len(o) */ - int is_meth = is_method(stack_pointer, oparg); + int is_meth = method != NULL; int total_args = oparg + is_meth; DEOPT_IF(total_args != 1, CALL); - PyObject *callable = PEEK(total_args + 1); + if (is_meth) { + callable = method; + } PyInterpreterState *interp = _PyInterpreterState_GET(); DEOPT_IF(callable != interp->callable_cache.len, CALL); STAT_INC(CALL, hit); - PyObject *arg = TOP(); + PyObject *arg = args[-is_meth]; Py_ssize_t len_i = PyObject_Length(arg); if (len_i < 0) { goto error; } - PyObject *res = PyLong_FromSsize_t(len_i); + res = PyLong_FromSsize_t(len_i); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - STACK_SHRINK(2-is_meth); - SET_TOP(res); Py_DECREF(callable); Py_DECREF(arg); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(4); DISPATCH(); } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index e438c991fc0194..55ffb66496a295 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -309,7 +309,7 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { case CALL_BUILTIN_FAST_WITH_KEYWORDS: return oparg + 2; case CALL_NO_KW_LEN: - return -1; + return oparg + 2; case CALL_NO_KW_ISINSTANCE: return -1; case CALL_NO_KW_LIST_APPEND: @@ -655,7 +655,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case CALL_BUILTIN_FAST_WITH_KEYWORDS: return 1; case CALL_NO_KW_LEN: - return -1; + return 1; case CALL_NO_KW_ISINSTANCE: return -1; case CALL_NO_KW_LIST_APPEND: @@ -853,7 +853,7 @@ struct opcode_metadata { [CALL_NO_KW_BUILTIN_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_BUILTIN_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_LEN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [CALL_NO_KW_LEN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_ISINSTANCE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_LIST_APPEND] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_METHOD_DESCRIPTOR_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, From 4397083897eb0f0e23ddb7d21ab29bf18c7db5bc Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sat, 4 Feb 2023 14:38:44 -0800 Subject: [PATCH 16/39] Modernize CALL_NO_KW_ISINSTANCE --- Python/bytecodes.c | 29 +++++++++++++---------------- Python/generated_cases.c.h | 32 +++++++++++++++++++------------- Python/opcode_metadata.h | 6 +++--- 3 files changed, 35 insertions(+), 32 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 1fc926eb74497f..9e84e431c128df 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2367,7 +2367,7 @@ dummy_func( CALL_NO_KW_BUILTIN_FAST, CALL_BUILTIN_FAST_WITH_KEYWORDS, CALL_NO_KW_LEN, - // CALL_NO_KW_ISINSTANCE, + CALL_NO_KW_ISINSTANCE, // CALL_NO_KW_LIST_APPEND, // CALL_NO_KW_METHOD_DESCRIPTOR_O, // CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, @@ -2725,37 +2725,34 @@ dummy_func( ERROR_IF(res == NULL, error); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_NO_KW_ISINSTANCE) { + inst(CALL_NO_KW_ISINSTANCE, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) { assert(cframe.use_tracing == 0); assert(kwnames == NULL); /* isinstance(o, o2) */ - int is_meth = is_method(stack_pointer, oparg); - int total_args = oparg + is_meth; - PyObject *callable = PEEK(total_args + 1); + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + callable = method; + args--; + total_args++; + } DEOPT_IF(total_args != 2, CALL); PyInterpreterState *interp = _PyInterpreterState_GET(); DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); STAT_INC(CALL, hit); - PyObject *cls = POP(); - PyObject *inst = TOP(); + PyObject *cls = args[1]; + PyObject *inst = args[0]; int retval = PyObject_IsInstance(inst, cls); if (retval < 0) { - Py_DECREF(cls); goto error; } - PyObject *res = PyBool_FromLong(retval); + res = PyBool_FromLong(retval); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - STACK_SHRINK(2-is_meth); - SET_TOP(res); Py_DECREF(inst); Py_DECREF(cls); Py_DECREF(callable); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + ERROR_IF(res == NULL, error); } // stack effect: (__0, __array[oparg] -- ) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6a3ac6d7c3cdf4..32a67849a845e7 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3391,35 +3391,41 @@ } TARGET(CALL_NO_KW_ISINSTANCE) { + PyObject **args = &PEEK(oparg); + PyObject *callable = PEEK(1 + oparg); + PyObject *method = PEEK(2 + oparg); + PyObject *res; assert(cframe.use_tracing == 0); assert(kwnames == NULL); /* isinstance(o, o2) */ - int is_meth = is_method(stack_pointer, oparg); - int total_args = oparg + is_meth; - PyObject *callable = PEEK(total_args + 1); + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + callable = method; + args--; + total_args++; + } DEOPT_IF(total_args != 2, CALL); PyInterpreterState *interp = _PyInterpreterState_GET(); DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); STAT_INC(CALL, hit); - PyObject *cls = POP(); - PyObject *inst = TOP(); + PyObject *cls = args[1]; + PyObject *inst = args[0]; int retval = PyObject_IsInstance(inst, cls); if (retval < 0) { - Py_DECREF(cls); goto error; } - PyObject *res = PyBool_FromLong(retval); + res = PyBool_FromLong(retval); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - STACK_SHRINK(2-is_meth); - SET_TOP(res); Py_DECREF(inst); Py_DECREF(cls); Py_DECREF(callable); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(4); DISPATCH(); } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 55ffb66496a295..c1dfb769c84b9f 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -311,7 +311,7 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { case CALL_NO_KW_LEN: return oparg + 2; case CALL_NO_KW_ISINSTANCE: - return -1; + return oparg + 2; case CALL_NO_KW_LIST_APPEND: return -1; case CALL_NO_KW_METHOD_DESCRIPTOR_O: @@ -657,7 +657,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case CALL_NO_KW_LEN: return 1; case CALL_NO_KW_ISINSTANCE: - return -1; + return 1; case CALL_NO_KW_LIST_APPEND: return -1; case CALL_NO_KW_METHOD_DESCRIPTOR_O: @@ -854,7 +854,7 @@ struct opcode_metadata { [CALL_NO_KW_BUILTIN_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_LEN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_ISINSTANCE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [CALL_NO_KW_ISINSTANCE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_LIST_APPEND] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_METHOD_DESCRIPTOR_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, From 8de596876c97dbed4c5a65ef877d62352626155e Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sat, 4 Feb 2023 15:04:12 -0800 Subject: [PATCH 17/39] Modernize CALL_NO_KW_LIST_APPEND --- Python/bytecodes.c | 25 ++++++++++++------------- Python/generated_cases.c.h | 22 ++++++++++++---------- Python/opcode_metadata.h | 6 +++--- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 9e84e431c128df..8b1e2b245dece9 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2368,7 +2368,7 @@ dummy_func( CALL_BUILTIN_FAST_WITH_KEYWORDS, CALL_NO_KW_LEN, CALL_NO_KW_ISINSTANCE, - // CALL_NO_KW_LIST_APPEND, + CALL_NO_KW_LIST_APPEND, // CALL_NO_KW_METHOD_DESCRIPTOR_O, // CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, // CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, @@ -2755,27 +2755,26 @@ dummy_func( ERROR_IF(res == NULL, error); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_NO_KW_LIST_APPEND) { + // This is secretly a super-instruction + inst(CALL_NO_KW_LIST_APPEND, (unused/1, unused/2, unused/1, method, self, args[oparg] -- res)) { assert(cframe.use_tracing == 0); assert(kwnames == NULL); assert(oparg == 1); - PyObject *callable = PEEK(3); + assert(method != NULL); PyInterpreterState *interp = _PyInterpreterState_GET(); - DEOPT_IF(callable != interp->callable_cache.list_append, CALL); - PyObject *list = SECOND(); - DEOPT_IF(!PyList_Check(list), CALL); + DEOPT_IF(method != interp->callable_cache.list_append, CALL); + DEOPT_IF(!PyList_Check(self), CALL); STAT_INC(CALL, hit); - PyObject *arg = POP(); - if (_PyList_AppendTakeRef((PyListObject *)list, arg) < 0) { - goto error; + if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { + goto pop_1_error; // Since arg is DECREF'ed already } - STACK_SHRINK(2); - Py_DECREF(list); - Py_DECREF(callable); + Py_DECREF(self); + Py_DECREF(method); + STACK_SHRINK(3); // CALL + POP_TOP JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); assert(_Py_OPCODE(next_instr[-1]) == POP_TOP); + DISPATCH(); } // stack effect: (__0, __array[oparg] -- ) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 32a67849a845e7..c841d3303378a8 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3430,22 +3430,24 @@ } TARGET(CALL_NO_KW_LIST_APPEND) { + PyObject **args = &PEEK(oparg); + PyObject *self = PEEK(1 + oparg); + PyObject *method = PEEK(2 + oparg); + PyObject *res; assert(cframe.use_tracing == 0); assert(kwnames == NULL); assert(oparg == 1); - PyObject *callable = PEEK(3); + assert(method != NULL); PyInterpreterState *interp = _PyInterpreterState_GET(); - DEOPT_IF(callable != interp->callable_cache.list_append, CALL); - PyObject *list = SECOND(); - DEOPT_IF(!PyList_Check(list), CALL); + DEOPT_IF(method != interp->callable_cache.list_append, CALL); + DEOPT_IF(!PyList_Check(self), CALL); STAT_INC(CALL, hit); - PyObject *arg = POP(); - if (_PyList_AppendTakeRef((PyListObject *)list, arg) < 0) { - goto error; + if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { + goto pop_1_error; // Since arg is DECREF'ed already } - STACK_SHRINK(2); - Py_DECREF(list); - Py_DECREF(callable); + Py_DECREF(self); + Py_DECREF(method); + STACK_SHRINK(3); // CALL + POP_TOP JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); assert(_Py_OPCODE(next_instr[-1]) == POP_TOP); diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index c1dfb769c84b9f..b73d1ccd09a001 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -313,7 +313,7 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { case CALL_NO_KW_ISINSTANCE: return oparg + 2; case CALL_NO_KW_LIST_APPEND: - return -1; + return oparg + 2; case CALL_NO_KW_METHOD_DESCRIPTOR_O: return -1; case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: @@ -659,7 +659,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case CALL_NO_KW_ISINSTANCE: return 1; case CALL_NO_KW_LIST_APPEND: - return -1; + return 1; case CALL_NO_KW_METHOD_DESCRIPTOR_O: return -1; case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: @@ -855,7 +855,7 @@ struct opcode_metadata { [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_LEN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_ISINSTANCE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_LIST_APPEND] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [CALL_NO_KW_LIST_APPEND] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_METHOD_DESCRIPTOR_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, From 05b3fc7c5d1dcb0f507424671bafd9632a66a759 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sat, 4 Feb 2023 15:20:47 -0800 Subject: [PATCH 18/39] Modernize CALL_NO_KW_METHOD_DESCRIPTOR_O --- Python/bytecodes.c | 26 ++++++++++++-------------- Python/generated_cases.c.h | 28 +++++++++++++++++----------- Python/opcode_metadata.h | 6 +++--- 3 files changed, 32 insertions(+), 28 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8b1e2b245dece9..c9b606d9ddda2f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2369,7 +2369,7 @@ dummy_func( CALL_NO_KW_LEN, CALL_NO_KW_ISINSTANCE, CALL_NO_KW_LIST_APPEND, - // CALL_NO_KW_METHOD_DESCRIPTOR_O, + CALL_NO_KW_METHOD_DESCRIPTOR_O, // CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, // CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, // CALL_NO_KW_METHOD_DESCRIPTOR_FAST, @@ -2777,19 +2777,22 @@ dummy_func( DISPATCH(); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_NO_KW_METHOD_DESCRIPTOR_O) { + inst(CALL_NO_KW_METHOD_DESCRIPTOR_O, (unused/1, unused/2, unused/1, method, unused, args[oparg] -- res)) { assert(kwnames == NULL); - int is_meth = is_method(stack_pointer, oparg); - int total_args = oparg + is_meth; + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + args--; + total_args++; + } PyMethodDescrObject *callable = (PyMethodDescrObject *)PEEK(total_args + 1); DEOPT_IF(total_args != 2, CALL); DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); PyMethodDef *meth = callable->d_method; DEOPT_IF(meth->ml_flags != METH_O, CALL); - PyObject *arg = TOP(); - PyObject *self = SECOND(); + PyObject *arg = args[1]; + PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; @@ -2798,18 +2801,13 @@ dummy_func( if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { goto error; } - PyObject *res = _PyCFunction_TrampolineCall(cfunc, self, arg); + res = _PyCFunction_TrampolineCall(cfunc, self, arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); Py_DECREF(arg); - STACK_SHRINK(oparg + 1); - SET_TOP(res); Py_DECREF(callable); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + ERROR_IF(res == NULL, error); CHECK_EVAL_BREAKER(); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index c841d3303378a8..54d804995d46c2 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3455,17 +3455,24 @@ } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { + PyObject **args = &PEEK(oparg); + PyObject *method = PEEK(2 + oparg); + PyObject *res; assert(kwnames == NULL); - int is_meth = is_method(stack_pointer, oparg); - int total_args = oparg + is_meth; + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + args--; + total_args++; + } PyMethodDescrObject *callable = (PyMethodDescrObject *)PEEK(total_args + 1); DEOPT_IF(total_args != 2, CALL); DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); PyMethodDef *meth = callable->d_method; DEOPT_IF(meth->ml_flags != METH_O, CALL); - PyObject *arg = TOP(); - PyObject *self = SECOND(); + PyObject *arg = args[1]; + PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; @@ -3474,18 +3481,17 @@ if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { goto error; } - PyObject *res = _PyCFunction_TrampolineCall(cfunc, self, arg); + res = _PyCFunction_TrampolineCall(cfunc, self, arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); Py_DECREF(arg); - STACK_SHRINK(oparg + 1); - SET_TOP(res); Py_DECREF(callable); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(4); CHECK_EVAL_BREAKER(); DISPATCH(); } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index b73d1ccd09a001..8a97d3287b0ccc 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -315,7 +315,7 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { case CALL_NO_KW_LIST_APPEND: return oparg + 2; case CALL_NO_KW_METHOD_DESCRIPTOR_O: - return -1; + return oparg + 2; case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: return -1; case CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS: @@ -661,7 +661,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case CALL_NO_KW_LIST_APPEND: return 1; case CALL_NO_KW_METHOD_DESCRIPTOR_O: - return -1; + return 1; case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: return -1; case CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS: @@ -856,7 +856,7 @@ struct opcode_metadata { [CALL_NO_KW_LEN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_ISINSTANCE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_LIST_APPEND] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_METHOD_DESCRIPTOR_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [CALL_NO_KW_METHOD_DESCRIPTOR_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, From b834ab2a56b50c6badf459be98cdb5d9e9b4e89e Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sat, 4 Feb 2023 15:54:24 -0800 Subject: [PATCH 19/39] Modernize CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS --- Python/bytecodes.c | 17 ++++++++--------- Python/generated_cases.c.h | 14 ++++++++------ Python/opcode_metadata.h | 6 +++--- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c9b606d9ddda2f..fdc6d59b375377 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2370,7 +2370,7 @@ dummy_func( CALL_NO_KW_ISINSTANCE, CALL_NO_KW_LIST_APPEND, CALL_NO_KW_METHOD_DESCRIPTOR_O, - // CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, // CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, // CALL_NO_KW_METHOD_DESCRIPTOR_FAST, }; @@ -2811,9 +2811,8 @@ dummy_func( CHECK_EVAL_BREAKER(); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { - int is_meth = is_method(stack_pointer, oparg); + inst(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (unused/1, unused/2, unused/1, method, unused, args[oparg] -- res)) { + int is_meth = method != NULL; int total_args = oparg + is_meth; PyMethodDescrObject *callable = (PyMethodDescrObject *)PEEK(total_args + 1); @@ -2821,15 +2820,14 @@ dummy_func( PyMethodDef *meth = callable->d_method; DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); PyTypeObject *d_type = callable->d_common.d_type; - PyObject *self = PEEK(total_args); + PyObject *self = args[-is_meth]; DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); STAT_INC(CALL, hit); - int nargs = total_args-1; + int nargs = total_args - 1; STACK_SHRINK(nargs); _PyCFunctionFastWithKeywords cfunc = (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - PyObject *res = cfunc(self, stack_pointer, nargs - KWNAMES_LEN(), - kwnames); + res = cfunc(self, stack_pointer, nargs - KWNAMES_LEN(), kwnames); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); kwnames = NULL; @@ -2838,7 +2836,7 @@ dummy_func( Py_DECREF(stack_pointer[i]); } Py_DECREF(self); - STACK_SHRINK(2-is_meth); + STACK_SHRINK(2 - is_meth); SET_TOP(res); Py_DECREF(callable); if (res == NULL) { @@ -2846,6 +2844,7 @@ dummy_func( } JUMPBY(INLINE_CACHE_ENTRIES_CALL); CHECK_EVAL_BREAKER(); + DISPATCH(); } // stack effect: (__0, __array[oparg] -- ) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 54d804995d46c2..a62fb3ffbf6241 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3497,7 +3497,10 @@ } TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { - int is_meth = is_method(stack_pointer, oparg); + PyObject **args = &PEEK(oparg); + PyObject *method = PEEK(2 + oparg); + PyObject *res; + int is_meth = method != NULL; int total_args = oparg + is_meth; PyMethodDescrObject *callable = (PyMethodDescrObject *)PEEK(total_args + 1); @@ -3505,15 +3508,14 @@ PyMethodDef *meth = callable->d_method; DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); PyTypeObject *d_type = callable->d_common.d_type; - PyObject *self = PEEK(total_args); + PyObject *self = args[-is_meth]; DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); STAT_INC(CALL, hit); - int nargs = total_args-1; + int nargs = total_args - 1; STACK_SHRINK(nargs); _PyCFunctionFastWithKeywords cfunc = (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - PyObject *res = cfunc(self, stack_pointer, nargs - KWNAMES_LEN(), - kwnames); + res = cfunc(self, stack_pointer, nargs - KWNAMES_LEN(), kwnames); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); kwnames = NULL; @@ -3522,7 +3524,7 @@ Py_DECREF(stack_pointer[i]); } Py_DECREF(self); - STACK_SHRINK(2-is_meth); + STACK_SHRINK(2 - is_meth); SET_TOP(res); Py_DECREF(callable); if (res == NULL) { diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 8a97d3287b0ccc..4a1a22bf80e745 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -317,7 +317,7 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { case CALL_NO_KW_METHOD_DESCRIPTOR_O: return oparg + 2; case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: - return -1; + return oparg + 2; case CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS: return -1; case CALL_NO_KW_METHOD_DESCRIPTOR_FAST: @@ -663,7 +663,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case CALL_NO_KW_METHOD_DESCRIPTOR_O: return 1; case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: - return -1; + return 1; case CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS: return -1; case CALL_NO_KW_METHOD_DESCRIPTOR_FAST: @@ -857,7 +857,7 @@ struct opcode_metadata { [CALL_NO_KW_ISINSTANCE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_LIST_APPEND] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_METHOD_DESCRIPTOR_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_FUNCTION_EX] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, From d2c72d167ff9a206f1db47569fd905b91e2197fd Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sat, 4 Feb 2023 15:57:48 -0800 Subject: [PATCH 20/39] Modernize CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS --- Python/bytecodes.c | 12 ++++++------ Python/generated_cases.c.h | 9 ++++++--- Python/opcode_metadata.h | 6 +++--- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index fdc6d59b375377..5633ad85a135ed 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2371,7 +2371,7 @@ dummy_func( CALL_NO_KW_LIST_APPEND, CALL_NO_KW_METHOD_DESCRIPTOR_O, CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, - // CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, + CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, // CALL_NO_KW_METHOD_DESCRIPTOR_FAST, }; @@ -2847,17 +2847,16 @@ dummy_func( DISPATCH(); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) { + inst(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, (unused/1, unused/2, unused/1, method, unused, args[oparg] -- res)) { assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); - int is_meth = is_method(stack_pointer, oparg); + int is_meth = method != NULL; int total_args = oparg + is_meth; DEOPT_IF(total_args != 1, CALL); PyMethodDescrObject *callable = (PyMethodDescrObject *)SECOND(); DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); PyMethodDef *meth = callable->d_method; - PyObject *self = TOP(); + PyObject *self = args[-is_meth]; DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); STAT_INC(CALL, hit); @@ -2867,7 +2866,7 @@ dummy_func( if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { goto error; } - PyObject *res = _PyCFunction_TrampolineCall(cfunc, self, NULL); + res = _PyCFunction_TrampolineCall(cfunc, self, NULL); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); @@ -2879,6 +2878,7 @@ dummy_func( } JUMPBY(INLINE_CACHE_ENTRIES_CALL); CHECK_EVAL_BREAKER(); + DISPATCH(); } // stack effect: (__0, __array[oparg] -- ) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index a62fb3ffbf6241..bed7bd0cc84d2b 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3536,15 +3536,18 @@ } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) { + PyObject **args = &PEEK(oparg); + PyObject *method = PEEK(2 + oparg); + PyObject *res; assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); - int is_meth = is_method(stack_pointer, oparg); + int is_meth = method != NULL; int total_args = oparg + is_meth; DEOPT_IF(total_args != 1, CALL); PyMethodDescrObject *callable = (PyMethodDescrObject *)SECOND(); DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); PyMethodDef *meth = callable->d_method; - PyObject *self = TOP(); + PyObject *self = args[-is_meth]; DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); STAT_INC(CALL, hit); @@ -3554,7 +3557,7 @@ if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { goto error; } - PyObject *res = _PyCFunction_TrampolineCall(cfunc, self, NULL); + res = _PyCFunction_TrampolineCall(cfunc, self, NULL); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 4a1a22bf80e745..a60c9b3140eef7 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -319,7 +319,7 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: return oparg + 2; case CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS: - return -1; + return oparg + 2; case CALL_NO_KW_METHOD_DESCRIPTOR_FAST: return -1; case CALL_FUNCTION_EX: @@ -665,7 +665,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: return 1; case CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS: - return -1; + return 1; case CALL_NO_KW_METHOD_DESCRIPTOR_FAST: return -1; case CALL_FUNCTION_EX: @@ -858,7 +858,7 @@ struct opcode_metadata { [CALL_NO_KW_LIST_APPEND] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_METHOD_DESCRIPTOR_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_FUNCTION_EX] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [MAKE_FUNCTION] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, From cbd3fe9a215d6749ae70efd533ec927dc0b1f229 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sat, 4 Feb 2023 16:00:09 -0800 Subject: [PATCH 21/39] Modernize CALL_NO_KW_METHOD_DESCRIPTOR_FAST --- Python/bytecodes.c | 12 ++++++------ Python/generated_cases.c.h | 9 ++++++--- Python/opcode_metadata.h | 6 +++--- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5633ad85a135ed..e21f7efe254276 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2372,7 +2372,7 @@ dummy_func( CALL_NO_KW_METHOD_DESCRIPTOR_O, CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, - // CALL_NO_KW_METHOD_DESCRIPTOR_FAST, + CALL_NO_KW_METHOD_DESCRIPTOR_FAST, }; // Stack is either @@ -2881,10 +2881,9 @@ dummy_func( DISPATCH(); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) { + inst(CALL_NO_KW_METHOD_DESCRIPTOR_FAST, (unused/1, unused/2, unused/1, method, unused, args[oparg] -- res)) { assert(kwnames == NULL); - int is_meth = is_method(stack_pointer, oparg); + int is_meth = method != NULL; int total_args = oparg + is_meth; PyMethodDescrObject *callable = (PyMethodDescrObject *)PEEK(total_args + 1); @@ -2899,14 +2898,14 @@ dummy_func( (_PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args-1; STACK_SHRINK(nargs); - PyObject *res = cfunc(self, stack_pointer, nargs); + res = cfunc(self, stack_pointer, nargs); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Clear the stack of the arguments. */ for (int i = 0; i < nargs; i++) { Py_DECREF(stack_pointer[i]); } Py_DECREF(self); - STACK_SHRINK(2-is_meth); + STACK_SHRINK(2 - is_meth); SET_TOP(res); Py_DECREF(callable); if (res == NULL) { @@ -2914,6 +2913,7 @@ dummy_func( } JUMPBY(INLINE_CACHE_ENTRIES_CALL); CHECK_EVAL_BREAKER(); + DISPATCH(); } // error: CALL_FUNCTION_EX has irregular stack effect diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index bed7bd0cc84d2b..b2421a98eb6353 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3573,8 +3573,11 @@ } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) { + PyObject **args = &PEEK(oparg); + PyObject *method = PEEK(2 + oparg); + PyObject *res; assert(kwnames == NULL); - int is_meth = is_method(stack_pointer, oparg); + int is_meth = method != NULL; int total_args = oparg + is_meth; PyMethodDescrObject *callable = (PyMethodDescrObject *)PEEK(total_args + 1); @@ -3589,14 +3592,14 @@ (_PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args-1; STACK_SHRINK(nargs); - PyObject *res = cfunc(self, stack_pointer, nargs); + res = cfunc(self, stack_pointer, nargs); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Clear the stack of the arguments. */ for (int i = 0; i < nargs; i++) { Py_DECREF(stack_pointer[i]); } Py_DECREF(self); - STACK_SHRINK(2-is_meth); + STACK_SHRINK(2 - is_meth); SET_TOP(res); Py_DECREF(callable); if (res == NULL) { diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index a60c9b3140eef7..881e99792eeb9b 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -321,7 +321,7 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { case CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS: return oparg + 2; case CALL_NO_KW_METHOD_DESCRIPTOR_FAST: - return -1; + return oparg + 2; case CALL_FUNCTION_EX: return -1; case MAKE_FUNCTION: @@ -667,7 +667,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS: return 1; case CALL_NO_KW_METHOD_DESCRIPTOR_FAST: - return -1; + return 1; case CALL_FUNCTION_EX: return -1; case MAKE_FUNCTION: @@ -859,7 +859,7 @@ struct opcode_metadata { [CALL_NO_KW_METHOD_DESCRIPTOR_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_FUNCTION_EX] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [MAKE_FUNCTION] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [RETURN_GENERATOR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, From ff6666fe0099488f822c99fb80c87747d0b71769 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 7 Feb 2023 15:42:40 -0800 Subject: [PATCH 22/39] Brandt's first review Co-authored-by: Brandt Bucher --- Python/bytecodes.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 77ff7d1727b643..c6ef534fd08060 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2482,7 +2482,6 @@ dummy_func( inst(CALL_PY_EXACT_ARGS, (unused/1, func_version/2, unused/1, thing1, thing2, unused[oparg] -- unused)) { assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); - // _PyCallCache *cache = (_PyCallCache *)next_instr; int is_meth = thing1 != NULL; int argcount = oparg + is_meth; PyObject *callable = is_meth ? thing1 : thing2; @@ -2755,7 +2754,7 @@ dummy_func( } // This is secretly a super-instruction - inst(CALL_NO_KW_LIST_APPEND, (unused/1, unused/2, unused/1, method, self, args[oparg] -- res)) { + inst(CALL_NO_KW_LIST_APPEND, (unused/1, unused/2, unused/1, method, self, args[oparg] -- unused)) { assert(cframe.use_tracing == 0); assert(kwnames == NULL); assert(oparg == 1); @@ -2880,7 +2879,7 @@ dummy_func( DISPATCH(); } - inst(CALL_NO_KW_METHOD_DESCRIPTOR_FAST, (unused/1, unused/2, unused/1, method, unused, args[oparg] -- res)) { + inst(CALL_NO_KW_METHOD_DESCRIPTOR_FAST, (unused/1, unused/2, unused/1, method, unused, unused[oparg] -- res)) { assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg + is_meth; From b9dd29e4ee75678341658506caecdb3949d52516 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 7 Feb 2023 15:31:49 -0800 Subject: [PATCH 23/39] Remove unused is_method() macro --- Python/ceval.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index ecb5bf9655553e..1f3f875d9c0aac 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -688,9 +688,6 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { } -// GH-89279: Must be a macro to be sure it's inlined by MSVC. -#define is_method(stack_pointer, args) (PEEK((args)+2) != NULL) - #define KWNAMES_LEN() \ (kwnames == NULL ? 0 : ((int)PyTuple_GET_SIZE(kwnames))) From 6830fd8f79295fda91d563be86c1583f35bab188 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 7 Feb 2023 15:43:33 -0800 Subject: [PATCH 24/39] Regenerate code --- Python/generated_cases.c.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index bdd5b1a93f1738..cb6f84687df79d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3091,7 +3091,6 @@ uint32_t func_version = read_u32(&next_instr[1].cache); assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); - // _PyCallCache *cache = (_PyCallCache *)next_instr; int is_meth = thing1 != NULL; int argcount = oparg + is_meth; PyObject *callable = is_meth ? thing1 : thing2; @@ -3452,7 +3451,6 @@ PyObject **args = &PEEK(oparg); PyObject *self = PEEK(1 + oparg); PyObject *method = PEEK(2 + oparg); - PyObject *res; assert(cframe.use_tracing == 0); assert(kwnames == NULL); assert(oparg == 1); @@ -3592,7 +3590,6 @@ } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) { - PyObject **args = &PEEK(oparg); PyObject *method = PEEK(2 + oparg); PyObject *res; assert(kwnames == NULL); From 448776bcec66ad002408105b4c8118e36ed1c02c Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 7 Feb 2023 15:53:16 -0800 Subject: [PATCH 25/39] Use Py_DECREF(&PyType_Type) after all --- Python/bytecodes.c | 2 +- Python/generated_cases.c.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c6ef534fd08060..b64f604329fd14 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2542,7 +2542,7 @@ dummy_func( STAT_INC(CALL, hit); res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); - Py_DECREF(callable); + Py_DECREF(&PyType_Type); // I.e., callable } inst(CALL_NO_KW_STR_1, (unused/1, unused/2, unused/1, null, callable, args[oparg] -- res)) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index cb6f84687df79d..dc4714408cb91c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3159,7 +3159,7 @@ STAT_INC(CALL, hit); res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); - Py_DECREF(callable); + Py_DECREF(&PyType_Type); // I.e., callable STACK_SHRINK(oparg); STACK_SHRINK(1); POKE(1, res); From b29fdc0ae61e974080ecd6b5a6c6aa7dbcdf29af Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 7 Feb 2023 15:59:41 -0800 Subject: [PATCH 26/39] Cleanup CALL_NO_KW_BUILTIN_O --- Python/bytecodes.c | 8 +++++--- Python/generated_cases.c.h | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b64f604329fd14..fc7563f9907a50 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2604,11 +2604,13 @@ dummy_func( /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = method != NULL; - int total_args = oparg + is_meth; - DEOPT_IF(total_args != 1, CALL); + int total_args = oparg; if (is_meth) { callable = method; + args--; + total_args++; } + DEOPT_IF(total_args != 1, CALL); DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); STAT_INC(CALL, hit); @@ -2618,7 +2620,7 @@ dummy_func( if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { goto error; } - PyObject *arg = args[-is_meth]; + PyObject *arg = args[0]; res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index dc4714408cb91c..6db09ecfab624f 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3257,11 +3257,13 @@ /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = method != NULL; - int total_args = oparg + is_meth; - DEOPT_IF(total_args != 1, CALL); + int total_args = oparg; if (is_meth) { callable = method; + args--; + total_args++; } + DEOPT_IF(total_args != 1, CALL); DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); STAT_INC(CALL, hit); @@ -3271,7 +3273,7 @@ if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { goto error; } - PyObject *arg = args[-is_meth]; + PyObject *arg = args[0]; res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); From b662db728f99dc563c1e8bf19897cafa0f0e22b5 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 7 Feb 2023 16:01:55 -0800 Subject: [PATCH 27/39] Cleanup CALL_NO_KW_BUILTIN_FAST --- Python/bytecodes.c | 8 +++++--- Python/generated_cases.c.h | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index fc7563f9907a50..7cd4a3486342cf 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2636,9 +2636,11 @@ dummy_func( /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = method != NULL; - int total_args = oparg + is_meth; + int total_args = oparg; if (is_meth) { callable = method; + args--; + total_args++; } DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); @@ -2647,13 +2649,13 @@ dummy_func( /* res = func(self, args, nargs) */ res = ((_PyCFunctionFast)(void(*)(void))cfunc)( PyCFunction_GET_SELF(callable), - args - is_meth, + args, total_args); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i - is_meth]); + Py_DECREF(args[i]); } Py_DECREF(callable); ERROR_IF(res == NULL, error); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6db09ecfab624f..0c85208c5a46a6 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3298,9 +3298,11 @@ /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = method != NULL; - int total_args = oparg + is_meth; + int total_args = oparg; if (is_meth) { callable = method; + args--; + total_args++; } DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); @@ -3309,13 +3311,13 @@ /* res = func(self, args, nargs) */ res = ((_PyCFunctionFast)(void(*)(void))cfunc)( PyCFunction_GET_SELF(callable), - args - is_meth, + args, total_args); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i - is_meth]); + Py_DECREF(args[i]); } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } From a9685455fe58ca68341854772afa0ee1471f31e9 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 7 Feb 2023 16:04:26 -0800 Subject: [PATCH 28/39] Cleanup CALL_BUILTIN_FAST_WITH_KEYWORDS --- Python/bytecodes.c | 8 +++++--- Python/generated_cases.c.h | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 7cd4a3486342cf..0187235d44e9d9 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2671,9 +2671,11 @@ dummy_func( assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; - int total_args = oparg + is_meth; + int total_args = oparg; if (is_meth) { callable = method; + args--; + total_args++; } DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); DEOPT_IF(PyCFunction_GET_FLAGS(callable) != @@ -2685,7 +2687,7 @@ dummy_func( PyCFunction_GET_FUNCTION(callable); res = cfunc( PyCFunction_GET_SELF(callable), - args - is_meth, + args, total_args - KWNAMES_LEN(), kwnames ); @@ -2694,7 +2696,7 @@ dummy_func( /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i - is_meth]); + Py_DECREF(args[i]); } Py_DECREF(callable); ERROR_IF(res == NULL, error); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 0c85208c5a46a6..d4de41ae52a0fa 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3342,9 +3342,11 @@ assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; - int total_args = oparg + is_meth; + int total_args = oparg; if (is_meth) { callable = method; + args--; + total_args++; } DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); DEOPT_IF(PyCFunction_GET_FLAGS(callable) != @@ -3356,7 +3358,7 @@ PyCFunction_GET_FUNCTION(callable); res = cfunc( PyCFunction_GET_SELF(callable), - args - is_meth, + args, total_args - KWNAMES_LEN(), kwnames ); @@ -3365,7 +3367,7 @@ /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i - is_meth]); + Py_DECREF(args[i]); } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } From 1de410f3208f741df5007602f2a2b5cd605b057c Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 7 Feb 2023 16:06:56 -0800 Subject: [PATCH 29/39] Cleanup CALL_NO_KW_LEN --- Python/bytecodes.c | 8 +++++--- Python/generated_cases.c.h | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0187235d44e9d9..0a6cade6fe91c2 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2708,15 +2708,17 @@ dummy_func( assert(kwnames == NULL); /* len(o) */ int is_meth = method != NULL; - int total_args = oparg + is_meth; - DEOPT_IF(total_args != 1, CALL); + int total_args = oparg; if (is_meth) { callable = method; + args--; + total_args++; } + DEOPT_IF(total_args != 1, CALL); PyInterpreterState *interp = _PyInterpreterState_GET(); DEOPT_IF(callable != interp->callable_cache.len, CALL); STAT_INC(CALL, hit); - PyObject *arg = args[-is_meth]; + PyObject *arg = args[0]; Py_ssize_t len_i = PyObject_Length(arg); if (len_i < 0) { goto error; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index d4de41ae52a0fa..4efb11648803af 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3388,15 +3388,17 @@ assert(kwnames == NULL); /* len(o) */ int is_meth = method != NULL; - int total_args = oparg + is_meth; - DEOPT_IF(total_args != 1, CALL); + int total_args = oparg; if (is_meth) { callable = method; + args--; + total_args++; } + DEOPT_IF(total_args != 1, CALL); PyInterpreterState *interp = _PyInterpreterState_GET(); DEOPT_IF(callable != interp->callable_cache.len, CALL); STAT_INC(CALL, hit); - PyObject *arg = args[-is_meth]; + PyObject *arg = args[0]; Py_ssize_t len_i = PyObject_Length(arg); if (len_i < 0) { goto error; From 390a9b43f1d85dc8c34d1129a397e311fc53558f Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 7 Feb 2023 18:15:38 -0800 Subject: [PATCH 30/39] Move KWNAMES_LEN() to ceval_macros.h --- Python/ceval.c | 3 --- Python/ceval_macros.h | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 1f3f875d9c0aac..75298efbcee9da 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -688,9 +688,6 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { } -#define KWNAMES_LEN() \ - (kwnames == NULL ? 0 : ((int)PyTuple_GET_SIZE(kwnames))) - /* Disable unused label warnings. They are handy for debugging, even if computed gotos aren't used. */ diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index d7a8f0beeec872..691bf8e1caae95 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -347,3 +347,6 @@ GETITEM(PyObject *v, Py_ssize_t i) { } while (0); #define NAME_ERROR_MSG "name '%.200s' is not defined" + +#define KWNAMES_LEN() \ + (kwnames == NULL ? 0 : ((int)PyTuple_GET_SIZE(kwnames))) From 425042012a53d5d675c531041490a29f60147adf Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 7 Feb 2023 18:27:03 -0800 Subject: [PATCH 31/39] Cleanup CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS --- Python/bytecodes.c | 24 ++++++++++-------------- Python/generated_cases.c.h | 27 ++++++++++++++------------- 2 files changed, 24 insertions(+), 27 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0a6cade6fe91c2..71b59dac636729 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2819,38 +2819,34 @@ dummy_func( inst(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (unused/1, unused/2, unused/1, method, unused, args[oparg] -- res)) { int is_meth = method != NULL; - int total_args = oparg + is_meth; + int total_args = oparg; + if (is_meth) { + args--; + total_args++; + } PyMethodDescrObject *callable = (PyMethodDescrObject *)PEEK(total_args + 1); DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); PyMethodDef *meth = callable->d_method; DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); PyTypeObject *d_type = callable->d_common.d_type; - PyObject *self = args[-is_meth]; + PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; - STACK_SHRINK(nargs); _PyCFunctionFastWithKeywords cfunc = (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - res = cfunc(self, stack_pointer, nargs - KWNAMES_LEN(), kwnames); + res = cfunc(self, args + 1, nargs - KWNAMES_LEN(), kwnames); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); kwnames = NULL; /* Free the arguments. */ - for (int i = 0; i < nargs; i++) { - Py_DECREF(stack_pointer[i]); + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); } - Py_DECREF(self); - STACK_SHRINK(2 - is_meth); - SET_TOP(res); Py_DECREF(callable); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + ERROR_IF(res == NULL, error); CHECK_EVAL_BREAKER(); - DISPATCH(); } inst(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, (unused/1, unused/2, unused/1, method, unused, args[oparg] -- res)) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 4efb11648803af..d81caf943c3744 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3526,36 +3526,37 @@ PyObject *method = PEEK(2 + oparg); PyObject *res; int is_meth = method != NULL; - int total_args = oparg + is_meth; + int total_args = oparg; + if (is_meth) { + args--; + total_args++; + } PyMethodDescrObject *callable = (PyMethodDescrObject *)PEEK(total_args + 1); DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); PyMethodDef *meth = callable->d_method; DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); PyTypeObject *d_type = callable->d_common.d_type; - PyObject *self = args[-is_meth]; + PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; - STACK_SHRINK(nargs); _PyCFunctionFastWithKeywords cfunc = (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - res = cfunc(self, stack_pointer, nargs - KWNAMES_LEN(), kwnames); + res = cfunc(self, args + 1, nargs - KWNAMES_LEN(), kwnames); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); kwnames = NULL; /* Free the arguments. */ - for (int i = 0; i < nargs; i++) { - Py_DECREF(stack_pointer[i]); + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); } - Py_DECREF(self); - STACK_SHRINK(2 - is_meth); - SET_TOP(res); Py_DECREF(callable); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(4); CHECK_EVAL_BREAKER(); DISPATCH(); } From 5ca2b9557c3c2944b0bbb6b44ef2990e5a1f137b Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 7 Feb 2023 18:37:51 -0800 Subject: [PATCH 32/39] Cleanup CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS --- Python/bytecodes.c | 16 +++++++--------- Python/generated_cases.c.h | 19 +++++++++++-------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 71b59dac636729..94091168cbfd4b 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2853,12 +2853,16 @@ dummy_func( assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; - int total_args = oparg + is_meth; + int total_args = oparg; + if (is_meth) { + args--; + total_args++; + } DEOPT_IF(total_args != 1, CALL); PyMethodDescrObject *callable = (PyMethodDescrObject *)SECOND(); DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); PyMethodDef *meth = callable->d_method; - PyObject *self = args[-is_meth]; + PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); STAT_INC(CALL, hit); @@ -2872,15 +2876,9 @@ dummy_func( _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); - STACK_SHRINK(oparg + 1); - SET_TOP(res); Py_DECREF(callable); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + ERROR_IF(res == NULL, error); CHECK_EVAL_BREAKER(); - DISPATCH(); } inst(CALL_NO_KW_METHOD_DESCRIPTOR_FAST, (unused/1, unused/2, unused/1, method, unused, unused[oparg] -- res)) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index d81caf943c3744..ea0bb7b737cbb6 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3568,12 +3568,16 @@ assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; - int total_args = oparg + is_meth; + int total_args = oparg; + if (is_meth) { + args--; + total_args++; + } DEOPT_IF(total_args != 1, CALL); PyMethodDescrObject *callable = (PyMethodDescrObject *)SECOND(); DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); PyMethodDef *meth = callable->d_method; - PyObject *self = args[-is_meth]; + PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); STAT_INC(CALL, hit); @@ -3587,13 +3591,12 @@ _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); - STACK_SHRINK(oparg + 1); - SET_TOP(res); Py_DECREF(callable); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(4); CHECK_EVAL_BREAKER(); DISPATCH(); } From 9c4171295998c3f7ad58fc4e29fef9c9d9897c48 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 7 Feb 2023 18:51:56 -0800 Subject: [PATCH 33/39] Cleanup CALL_NO_KW_METHOD_DESCRIPTOR_FAST --- Python/bytecodes.c | 28 ++++++++++++---------------- Python/generated_cases.c.h | 30 ++++++++++++++++-------------- 2 files changed, 28 insertions(+), 30 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 94091168cbfd4b..dd79609ed5ba71 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2881,39 +2881,35 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(CALL_NO_KW_METHOD_DESCRIPTOR_FAST, (unused/1, unused/2, unused/1, method, unused, unused[oparg] -- res)) { + inst(CALL_NO_KW_METHOD_DESCRIPTOR_FAST, (unused/1, unused/2, unused/1, method, unused, args[oparg] -- res)) { assert(kwnames == NULL); int is_meth = method != NULL; - int total_args = oparg + is_meth; + int total_args = oparg; + if (is_meth) { + args--; + total_args++; + } PyMethodDescrObject *callable = (PyMethodDescrObject *)PEEK(total_args + 1); /* Builtin METH_FASTCALL methods, without keywords */ DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); PyMethodDef *meth = callable->d_method; DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); - PyObject *self = PEEK(total_args); + PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); STAT_INC(CALL, hit); _PyCFunctionFast cfunc = (_PyCFunctionFast)(void(*)(void))meth->ml_meth; - int nargs = total_args-1; - STACK_SHRINK(nargs); - res = cfunc(self, stack_pointer, nargs); + int nargs = total_args - 1; + res = cfunc(self, args + 1, nargs); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Clear the stack of the arguments. */ - for (int i = 0; i < nargs; i++) { - Py_DECREF(stack_pointer[i]); + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); } - Py_DECREF(self); - STACK_SHRINK(2 - is_meth); - SET_TOP(res); Py_DECREF(callable); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + ERROR_IF(res == NULL, error); CHECK_EVAL_BREAKER(); - DISPATCH(); } // error: CALL_FUNCTION_EX has irregular stack effect diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ea0bb7b737cbb6..074af99506062c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3602,38 +3602,40 @@ } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) { + PyObject **args = &PEEK(oparg); PyObject *method = PEEK(2 + oparg); PyObject *res; assert(kwnames == NULL); int is_meth = method != NULL; - int total_args = oparg + is_meth; + int total_args = oparg; + if (is_meth) { + args--; + total_args++; + } PyMethodDescrObject *callable = (PyMethodDescrObject *)PEEK(total_args + 1); /* Builtin METH_FASTCALL methods, without keywords */ DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); PyMethodDef *meth = callable->d_method; DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); - PyObject *self = PEEK(total_args); + PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); STAT_INC(CALL, hit); _PyCFunctionFast cfunc = (_PyCFunctionFast)(void(*)(void))meth->ml_meth; - int nargs = total_args-1; - STACK_SHRINK(nargs); - res = cfunc(self, stack_pointer, nargs); + int nargs = total_args - 1; + res = cfunc(self, args + 1, nargs); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Clear the stack of the arguments. */ - for (int i = 0; i < nargs; i++) { - Py_DECREF(stack_pointer[i]); + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); } - Py_DECREF(self); - STACK_SHRINK(2 - is_meth); - SET_TOP(res); Py_DECREF(callable); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(4); CHECK_EVAL_BREAKER(); DISPATCH(); } From b7281a967e3d9c9e4a2094be74723c83339f5af7 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 7 Feb 2023 19:20:59 -0800 Subject: [PATCH 34/39] Cleanup CALL_PY_WITH_DEFAULTS --- Python/bytecodes.c | 19 +++++++++++-------- Python/generated_cases.c.h | 22 +++++++++++++--------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index dd79609ed5ba71..0952e64df22087 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2503,12 +2503,16 @@ dummy_func( DISPATCH_INLINED(new_frame); } - inst(CALL_PY_WITH_DEFAULTS, (unused/1, func_version/2, min_args/1, thing1, thing2, unused[oparg] -- unused)) { + inst(CALL_PY_WITH_DEFAULTS, (unused/1, func_version/2, min_args/1, method, callable, args[oparg] -- unused)) { assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); - int is_meth = thing1 != NULL; - int argcount = oparg + is_meth; - PyObject *callable = is_meth ? thing1 : thing2; + int is_meth = method != NULL; + int argcount = oparg; + if (is_meth) { + callable = method; + args--; + argcount++; + } DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; DEOPT_IF(func->func_version != func_version, CALL); @@ -2518,16 +2522,15 @@ dummy_func( DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); STAT_INC(CALL, hit); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(argcount); for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = stack_pointer[i]; + new_frame->localsplus[i] = args[i]; } for (int i = argcount; i < code->co_argcount; i++) { PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args); new_frame->localsplus[i] = Py_NewRef(def); } - STACK_SHRINK(2 - is_meth); + // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 074af99506062c..b5f28092354e34 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3113,15 +3113,20 @@ } TARGET(CALL_PY_WITH_DEFAULTS) { - PyObject *thing2 = PEEK(1 + oparg); - PyObject *thing1 = PEEK(2 + oparg); + PyObject **args = &PEEK(oparg); + PyObject *callable = PEEK(1 + oparg); + PyObject *method = PEEK(2 + oparg); uint32_t func_version = read_u32(&next_instr[1].cache); uint16_t min_args = read_u16(&next_instr[3].cache); assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); - int is_meth = thing1 != NULL; - int argcount = oparg + is_meth; - PyObject *callable = is_meth ? thing1 : thing2; + int is_meth = method != NULL; + int argcount = oparg; + if (is_meth) { + callable = method; + args--; + argcount++; + } DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; DEOPT_IF(func->func_version != func_version, CALL); @@ -3131,16 +3136,15 @@ DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); STAT_INC(CALL, hit); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(argcount); for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = stack_pointer[i]; + new_frame->localsplus[i] = args[i]; } for (int i = argcount; i < code->co_argcount; i++) { PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args); new_frame->localsplus[i] = Py_NewRef(def); } - STACK_SHRINK(2 - is_meth); + // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); } From e14087907325f66b8272c504cee12a9df824cd56 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 7 Feb 2023 19:26:13 -0800 Subject: [PATCH 35/39] Cleanup CALL_PY_EXACT_ARGS --- Python/bytecodes.c | 19 +++++++++++-------- Python/generated_cases.c.h | 22 +++++++++++++--------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0952e64df22087..b559a6293a09e7 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2479,12 +2479,16 @@ dummy_func( GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); } - inst(CALL_PY_EXACT_ARGS, (unused/1, func_version/2, unused/1, thing1, thing2, unused[oparg] -- unused)) { + inst(CALL_PY_EXACT_ARGS, (unused/1, func_version/2, unused/1, method, callable, args[oparg] -- unused)) { assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); - int is_meth = thing1 != NULL; - int argcount = oparg + is_meth; - PyObject *callable = is_meth ? thing1 : thing2; + int is_meth = method != NULL; + int argcount = oparg; + if (is_meth) { + callable = method; + args--; + argcount++; + } DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; DEOPT_IF(func->func_version != func_version, CALL); @@ -2493,12 +2497,11 @@ dummy_func( DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); STAT_INC(CALL, hit); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(argcount); for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = stack_pointer[i]; + new_frame->localsplus[i] = args[i]; } - STACK_SHRINK(2 - is_meth); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index b5f28092354e34..fcbd2cd8cf25e0 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3086,14 +3086,19 @@ TARGET(CALL_PY_EXACT_ARGS) { PREDICTED(CALL_PY_EXACT_ARGS); - PyObject *thing2 = PEEK(1 + oparg); - PyObject *thing1 = PEEK(2 + oparg); + PyObject **args = &PEEK(oparg); + PyObject *callable = PEEK(1 + oparg); + PyObject *method = PEEK(2 + oparg); uint32_t func_version = read_u32(&next_instr[1].cache); assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); - int is_meth = thing1 != NULL; - int argcount = oparg + is_meth; - PyObject *callable = is_meth ? thing1 : thing2; + int is_meth = method != NULL; + int argcount = oparg; + if (is_meth) { + callable = method; + args--; + argcount++; + } DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; DEOPT_IF(func->func_version != func_version, CALL); @@ -3102,12 +3107,11 @@ DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); STAT_INC(CALL, hit); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(argcount); for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = stack_pointer[i]; + new_frame->localsplus[i] = args[i]; } - STACK_SHRINK(2 - is_meth); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); } From 64073691cecc90b813f66a9eb02f6feee99f1ef8 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 7 Feb 2023 19:30:39 -0800 Subject: [PATCH 36/39] Cleanup CALL_BOUND_METHOD_EXACT_ARGS --- Python/bytecodes.c | 20 ++++++++++---------- Python/generated_cases.c.h | 18 +++++++++--------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b559a6293a09e7..0aee45b378b139 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2464,18 +2464,18 @@ dummy_func( DISPATCH(); // Prevents generator emitting the epologue. } - // Start out with [NULL, method, arg1, arg2, ...] - // Transform to [func, self, arg1, arg2, ...] + // Start out with [NULL, bound_method, arg1, arg2, ...] + // Transform to [callable, self, arg1, arg2, ...] // Then fall through to CALL_PY_EXACT_ARGS - inst(CALL_BOUND_METHOD_EXACT_ARGS, (unused/1, unused/2, unused/1, thing1, thing2, unused[oparg] -- unused)) { - DEOPT_IF(thing1 != NULL, CALL); - DEOPT_IF(Py_TYPE(thing2) != &PyMethod_Type, CALL); + inst(CALL_BOUND_METHOD_EXACT_ARGS, (unused/1, unused/2, unused/1, method, callable, unused[oparg] -- unused)) { + DEOPT_IF(method != NULL, CALL); + DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); - PyObject *self = ((PyMethodObject *)thing2)->im_self; - PEEK(oparg + 1) = Py_NewRef(self); // thing2 - PyObject *meth = ((PyMethodObject *)thing2)->im_func; - PEEK(oparg + 2) = Py_NewRef(meth); // thing1 - Py_DECREF(thing2); + PyObject *self = ((PyMethodObject *)callable)->im_self; + PEEK(oparg + 1) = Py_NewRef(self); // callable + PyObject *meth = ((PyMethodObject *)callable)->im_func; + PEEK(oparg + 2) = Py_NewRef(meth); // method + Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index fcbd2cd8cf25e0..2b9555dda0a3c1 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3071,16 +3071,16 @@ } TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { - PyObject *thing2 = PEEK(1 + oparg); - PyObject *thing1 = PEEK(2 + oparg); - DEOPT_IF(thing1 != NULL, CALL); - DEOPT_IF(Py_TYPE(thing2) != &PyMethod_Type, CALL); + PyObject *callable = PEEK(1 + oparg); + PyObject *method = PEEK(2 + oparg); + DEOPT_IF(method != NULL, CALL); + DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); - PyObject *self = ((PyMethodObject *)thing2)->im_self; - PEEK(oparg + 1) = Py_NewRef(self); // thing2 - PyObject *meth = ((PyMethodObject *)thing2)->im_func; - PEEK(oparg + 2) = Py_NewRef(meth); // thing1 - Py_DECREF(thing2); + PyObject *self = ((PyMethodObject *)callable)->im_self; + PEEK(oparg + 1) = Py_NewRef(self); // callable + PyObject *meth = ((PyMethodObject *)callable)->im_func; + PEEK(oparg + 2) = Py_NewRef(meth); // method + Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); } From b9432ba4fd32ae38092fefc4d215ad3743b0d1a9 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 7 Feb 2023 20:00:05 -0800 Subject: [PATCH 37/39] Cleanup CALL --- Python/bytecodes.c | 84 +++++++++++++++++--------------------- Python/generated_cases.c.h | 84 +++++++++++++++++++------------------- 2 files changed, 79 insertions(+), 89 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0aee45b378b139..7152ffe8ffcd9c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2374,56 +2374,58 @@ dummy_func( CALL_NO_KW_METHOD_DESCRIPTOR_FAST, }; - // Stack is either - // [NULL, function, arg1, arg2, ...] + // On entry, the stack is either + // [NULL, callable, arg1, arg2, ...] // or - // [method, self, arg1, arg2, ...] + // [method, self, arg1, arg2, ...] // (Some args may be keywords, see KW_NAMES, which sets 'kwnames'.) - // It will be replaced with [result]. - inst(CALL, (unused/1, unused/2, unused/1, thing1, thing2, unused[oparg] -- unused)) { + // On exit, the stack is [result]. + // When calling Python, inline the call using DISPATCH_INLINED(). + inst(CALL, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) { + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + callable = method; + args--; + total_args++; + } #if ENABLE_SPECIALIZATION _PyCallCache *cache = (_PyCallCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); - int is_meth = thing1 != NULL; - int nargs = oparg + is_meth; - PyObject *callable = PEEK(nargs + 1); next_instr--; - _Py_Specialize_Call(callable, next_instr, nargs, kwnames); + _Py_Specialize_Call(callable, next_instr, total_args, kwnames); DISPATCH_SAME_OPARG(); } STAT_INC(CALL, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ - int total_args, is_meth; - is_meth = thing1 != NULL; - PyObject *function = thing2; - if (!is_meth && Py_TYPE(function) == &PyMethod_Type) { - PyObject *self = ((PyMethodObject *)function)->im_self; - PEEK(oparg+1) = thing2 = Py_NewRef(self); - PyObject *meth = ((PyMethodObject *)function)->im_func; - PEEK(oparg+2) = thing1 = Py_NewRef(meth); - Py_DECREF(function); - is_meth = 1; - } - total_args = oparg + is_meth; - function = is_meth ? thing1 : thing2; + if (!is_meth && Py_TYPE(callable) == &PyMethod_Type) { + is_meth = 1; // For consistenct; it's dead, though + args--; + total_args++; + PyObject *self = ((PyMethodObject *)callable)->im_self; + args[0] = Py_NewRef(self); + method = ((PyMethodObject *)callable)->im_func; + args[-1] = Py_NewRef(method); + Py_DECREF(callable); + callable = method; + } int positional_args = total_args - KWNAMES_LEN(); // Check if the call can be inlined or not - if (Py_TYPE(function) == &PyFunction_Type && + if (Py_TYPE(callable) == &PyFunction_Type && tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)function)->vectorcall == _PyFunction_Vectorcall) + ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(function))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(function)); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(total_args); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)function, locals, - stack_pointer, positional_args, kwnames + tstate, (PyFunctionObject *)callable, locals, + args, positional_args, kwnames ); kwnames = NULL; - STACK_SHRINK(2 - is_meth); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { @@ -2433,35 +2435,25 @@ dummy_func( DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - PyObject *res; if (cframe.use_tracing) { res = trace_call_function( - tstate, function, stack_pointer - total_args, + tstate, callable, args, positional_args, kwnames); } else { res = PyObject_Vectorcall( - function, stack_pointer - total_args, + callable, args, positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); } kwnames = NULL; assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(function); - // Manipulate stack directly since we leave using DISPATCH(). - /* Clear the stack */ - STACK_SHRINK(total_args); + Py_DECREF(callable); for (int i = 0; i < total_args; i++) { - Py_DECREF(stack_pointer[i]); - } - STACK_SHRINK(2 - is_meth); - PUSH(res); - if (res == NULL) { - goto error; + Py_DECREF(args[i]); } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + ERROR_IF(res == NULL, error); CHECK_EVAL_BREAKER(); - DISPATCH(); // Prevents generator emitting the epologue. } // Start out with [NULL, bound_method, arg1, arg2, ...] diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2b9555dda0a3c1..2b7ca8859ca642 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2985,51 +2985,54 @@ TARGET(CALL) { PREDICTED(CALL); static_assert(INLINE_CACHE_ENTRIES_CALL == 4, "incorrect cache size"); - PyObject *thing2 = PEEK(1 + oparg); - PyObject *thing1 = PEEK(2 + oparg); + PyObject **args = &PEEK(oparg); + PyObject *callable = PEEK(1 + oparg); + PyObject *method = PEEK(2 + oparg); + PyObject *res; + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + callable = method; + args--; + total_args++; + } #if ENABLE_SPECIALIZATION _PyCallCache *cache = (_PyCallCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); - int is_meth = thing1 != NULL; - int nargs = oparg + is_meth; - PyObject *callable = PEEK(nargs + 1); next_instr--; - _Py_Specialize_Call(callable, next_instr, nargs, kwnames); + _Py_Specialize_Call(callable, next_instr, total_args, kwnames); DISPATCH_SAME_OPARG(); } STAT_INC(CALL, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ - int total_args, is_meth; - is_meth = thing1 != NULL; - PyObject *function = thing2; - if (!is_meth && Py_TYPE(function) == &PyMethod_Type) { - PyObject *self = ((PyMethodObject *)function)->im_self; - PEEK(oparg+1) = thing2 = Py_NewRef(self); - PyObject *meth = ((PyMethodObject *)function)->im_func; - PEEK(oparg+2) = thing1 = Py_NewRef(meth); - Py_DECREF(function); - is_meth = 1; - } - total_args = oparg + is_meth; - function = is_meth ? thing1 : thing2; + if (!is_meth && Py_TYPE(callable) == &PyMethod_Type) { + is_meth = 1; // For consistenct; it's dead, though + args--; + total_args++; + PyObject *self = ((PyMethodObject *)callable)->im_self; + args[0] = Py_NewRef(self); + method = ((PyMethodObject *)callable)->im_func; + args[-1] = Py_NewRef(method); + Py_DECREF(callable); + callable = method; + } int positional_args = total_args - KWNAMES_LEN(); // Check if the call can be inlined or not - if (Py_TYPE(function) == &PyFunction_Type && + if (Py_TYPE(callable) == &PyFunction_Type && tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)function)->vectorcall == _PyFunction_Vectorcall) + ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(function))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(function)); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(total_args); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)function, locals, - stack_pointer, positional_args, kwnames + tstate, (PyFunctionObject *)callable, locals, + args, positional_args, kwnames ); kwnames = NULL; - STACK_SHRINK(2 - is_meth); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { @@ -3039,35 +3042,30 @@ DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - PyObject *res; if (cframe.use_tracing) { res = trace_call_function( - tstate, function, stack_pointer - total_args, + tstate, callable, args, positional_args, kwnames); } else { res = PyObject_Vectorcall( - function, stack_pointer - total_args, + callable, args, positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); } kwnames = NULL; assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(function); - // Manipulate stack directly since we leave using DISPATCH(). - /* Clear the stack */ - STACK_SHRINK(total_args); + Py_DECREF(callable); for (int i = 0; i < total_args; i++) { - Py_DECREF(stack_pointer[i]); - } - STACK_SHRINK(2 - is_meth); - PUSH(res); - if (res == NULL) { - goto error; + Py_DECREF(args[i]); } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(4); CHECK_EVAL_BREAKER(); - DISPATCH(); // Prevents generator emitting the epologue. + DISPATCH(); } TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { From d84c30fec57e9db4b397ff6a2ee8ea5c5eb0d365 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 7 Feb 2023 20:26:07 -0800 Subject: [PATCH 38/39] Remove support for legacy instructions --- Tools/cases_generator/generate_cases.py | 19 +++++++------------ Tools/cases_generator/parser.py | 7 ++----- Tools/cases_generator/test_generator.py | 14 -------------- 3 files changed, 9 insertions(+), 31 deletions(-) diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 1fcfbb67709029..c646818a8e2077 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -198,7 +198,7 @@ class Instruction: # Parts of the underlying instruction definition inst: parser.InstDef register: bool - kind: typing.Literal["inst", "op", "legacy"] # Legacy means no (input -- output) + kind: typing.Literal["inst", "op"] name: str block: parser.Block block_text: list[str] # Block.text, less curlies, less PREDICT() calls @@ -838,8 +838,6 @@ def get_stack_effect_info( self, thing: parser.InstDef | parser.Super | parser.Macro ) -> tuple[AnyInstruction | None, str, str]: def effect_str(effects: list[StackEffect]) -> str: - if getattr(thing, "kind", None) == "legacy": - return str(-1) n_effect, sym_effect = list_effect_size(effects) if sym_effect: return f"{sym_effect} + {n_effect}" if n_effect else sym_effect @@ -966,15 +964,12 @@ def write_metadata(self) -> None: def write_metadata_for_inst(self, instr: Instruction) -> None: """Write metadata for a single instruction.""" dir_op1 = dir_op2 = dir_op3 = "DIR_NONE" - if instr.kind == "legacy": - assert not instr.register - else: - if instr.register: - directions: list[str] = [] - directions.extend("DIR_READ" for _ in instr.input_effects) - directions.extend("DIR_WRITE" for _ in instr.output_effects) - directions.extend("DIR_NONE" for _ in range(3)) - dir_op1, dir_op2, dir_op3 = directions[:3] + if instr.register: + directions: list[str] = [] + directions.extend("DIR_READ" for _ in instr.input_effects) + directions.extend("DIR_WRITE" for _ in instr.output_effects) + directions.extend("DIR_NONE" for _ in range(3)) + dir_op1, dir_op2, dir_op3 = directions[:3] self.out.emit( f" [{instr.name}] = {{ {dir_op1}, {dir_op2}, {dir_op3}, true, {INSTR_FMT_PREFIX}{instr.instr_fmt} }}," ) diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py index c7c8d8af6b7318..f02411c1b3b282 100644 --- a/Tools/cases_generator/parser.py +++ b/Tools/cases_generator/parser.py @@ -100,7 +100,7 @@ class OpName(Node): @dataclass class InstHeader(Node): register: bool - kind: Literal["inst", "op", "legacy"] # Legacy means no (inputs -- outputs) + kind: Literal["inst", "op"] name: str inputs: list[InputEffect] outputs: list[OutputEffect] @@ -109,7 +109,7 @@ class InstHeader(Node): @dataclass class InstDef(Node): register: bool - kind: Literal["inst", "op", "legacy"] + kind: Literal["inst", "op"] name: str inputs: list[InputEffect] outputs: list[OutputEffect] @@ -172,9 +172,6 @@ def inst_header(self) -> InstHeader | None: if self.expect(lx.RPAREN): if (tkn := self.peek()) and tkn.kind == lx.LBRACE: return InstHeader(register, kind, name, inp, outp) - elif self.expect(lx.RPAREN) and kind == "inst": - # No legacy stack effect if kind is "op". - return InstHeader(register, "legacy", name, [], []) return None def io_effect(self) -> tuple[list[InputEffect], list[OutputEffect]]: diff --git a/Tools/cases_generator/test_generator.py b/Tools/cases_generator/test_generator.py index 036094ac8ef487..412d3d14590050 100644 --- a/Tools/cases_generator/test_generator.py +++ b/Tools/cases_generator/test_generator.py @@ -62,20 +62,6 @@ def run_cases_test(input: str, expected: str): # print("End") assert actual.rstrip() == expected.rstrip() -def test_legacy(): - input = """ - inst(OP) { - spam(); - } - """ - output = """ - TARGET(OP) { - spam(); - DISPATCH(); - } - """ - run_cases_test(input, output) - def test_inst_no_args(): input = """ inst(OP, (--)) { From cd69634dab636ae273565f9952c3b944ca445743 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 8 Feb 2023 07:19:49 -0800 Subject: [PATCH 39/39] Restore support for legacy instructions It will be removed in a separate PR. This reverts commit d84c30fec57e9db4b397ff6a2ee8ea5c5eb0d365. --- Tools/cases_generator/generate_cases.py | 19 ++++++++++++------- Tools/cases_generator/parser.py | 7 +++++-- Tools/cases_generator/test_generator.py | 14 ++++++++++++++ 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index c646818a8e2077..1fcfbb67709029 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -198,7 +198,7 @@ class Instruction: # Parts of the underlying instruction definition inst: parser.InstDef register: bool - kind: typing.Literal["inst", "op"] + kind: typing.Literal["inst", "op", "legacy"] # Legacy means no (input -- output) name: str block: parser.Block block_text: list[str] # Block.text, less curlies, less PREDICT() calls @@ -838,6 +838,8 @@ def get_stack_effect_info( self, thing: parser.InstDef | parser.Super | parser.Macro ) -> tuple[AnyInstruction | None, str, str]: def effect_str(effects: list[StackEffect]) -> str: + if getattr(thing, "kind", None) == "legacy": + return str(-1) n_effect, sym_effect = list_effect_size(effects) if sym_effect: return f"{sym_effect} + {n_effect}" if n_effect else sym_effect @@ -964,12 +966,15 @@ def write_metadata(self) -> None: def write_metadata_for_inst(self, instr: Instruction) -> None: """Write metadata for a single instruction.""" dir_op1 = dir_op2 = dir_op3 = "DIR_NONE" - if instr.register: - directions: list[str] = [] - directions.extend("DIR_READ" for _ in instr.input_effects) - directions.extend("DIR_WRITE" for _ in instr.output_effects) - directions.extend("DIR_NONE" for _ in range(3)) - dir_op1, dir_op2, dir_op3 = directions[:3] + if instr.kind == "legacy": + assert not instr.register + else: + if instr.register: + directions: list[str] = [] + directions.extend("DIR_READ" for _ in instr.input_effects) + directions.extend("DIR_WRITE" for _ in instr.output_effects) + directions.extend("DIR_NONE" for _ in range(3)) + dir_op1, dir_op2, dir_op3 = directions[:3] self.out.emit( f" [{instr.name}] = {{ {dir_op1}, {dir_op2}, {dir_op3}, true, {INSTR_FMT_PREFIX}{instr.instr_fmt} }}," ) diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py index f02411c1b3b282..c7c8d8af6b7318 100644 --- a/Tools/cases_generator/parser.py +++ b/Tools/cases_generator/parser.py @@ -100,7 +100,7 @@ class OpName(Node): @dataclass class InstHeader(Node): register: bool - kind: Literal["inst", "op"] + kind: Literal["inst", "op", "legacy"] # Legacy means no (inputs -- outputs) name: str inputs: list[InputEffect] outputs: list[OutputEffect] @@ -109,7 +109,7 @@ class InstHeader(Node): @dataclass class InstDef(Node): register: bool - kind: Literal["inst", "op"] + kind: Literal["inst", "op", "legacy"] name: str inputs: list[InputEffect] outputs: list[OutputEffect] @@ -172,6 +172,9 @@ def inst_header(self) -> InstHeader | None: if self.expect(lx.RPAREN): if (tkn := self.peek()) and tkn.kind == lx.LBRACE: return InstHeader(register, kind, name, inp, outp) + elif self.expect(lx.RPAREN) and kind == "inst": + # No legacy stack effect if kind is "op". + return InstHeader(register, "legacy", name, [], []) return None def io_effect(self) -> tuple[list[InputEffect], list[OutputEffect]]: diff --git a/Tools/cases_generator/test_generator.py b/Tools/cases_generator/test_generator.py index 412d3d14590050..036094ac8ef487 100644 --- a/Tools/cases_generator/test_generator.py +++ b/Tools/cases_generator/test_generator.py @@ -62,6 +62,20 @@ def run_cases_test(input: str, expected: str): # print("End") assert actual.rstrip() == expected.rstrip() +def test_legacy(): + input = """ + inst(OP) { + spam(); + } + """ + output = """ + TARGET(OP) { + spam(); + DISPATCH(); + } + """ + run_cases_test(input, output) + def test_inst_no_args(): input = """ inst(OP, (--)) {