diff --git a/mypyc/analysis/blockfreq.py b/mypyc/analysis/blockfreq.py new file mode 100644 index 0000000000000..547fb9ce10d3f --- /dev/null +++ b/mypyc/analysis/blockfreq.py @@ -0,0 +1,32 @@ +"""Find basic blocks that are likely to be executed frequently. + +For example, this would not include blocks that have exception handlers. + +We can use different optimization heuristics for common and rare code. For +example, we can make IR fast to compile instead of fast to execute for rare +code. +""" + +from typing import Set + +from mypyc.ir.ops import BasicBlock, Goto, Branch + + +def frequently_executed_blocks(entry_point: BasicBlock) -> Set[BasicBlock]: + result: Set[BasicBlock] = set() + worklist = [entry_point] + while worklist: + block = worklist.pop() + if block in result: + continue + result.add(block) + t = block.terminator + if isinstance(t, Goto): + worklist.append(t.label) + elif isinstance(t, Branch): + if t.rare or t.traceback_entry is not None: + worklist.append(t.false) + else: + worklist.append(t.true) + worklist.append(t.false) + return result diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index 323d34f83a044..531121134b037 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -348,35 +348,56 @@ def declare_tuple_struct(self, tuple_type: RTuple) -> None: is_type=True, ) - def emit_inc_ref(self, dest: str, rtype: RType) -> None: + def emit_inc_ref(self, dest: str, rtype: RType, *, rare: bool = False) -> None: """Increment reference count of C expression `dest`. For composite unboxed structures (e.g. tuples) recursively increment reference counts for each component. + + If rare is True, optimize for code size and compilation speed. """ if is_int_rprimitive(rtype): - self.emit_line('CPyTagged_IncRef(%s);' % dest) + if rare: + self.emit_line('CPyTagged_IncRef(%s);' % dest) + else: + self.emit_line('CPyTagged_INCREF(%s);' % dest) elif isinstance(rtype, RTuple): for i, item_type in enumerate(rtype.types): self.emit_inc_ref('{}.f{}'.format(dest, i), item_type) elif not rtype.is_unboxed: + # Always inline, since this is a simple op self.emit_line('CPy_INCREF(%s);' % dest) # Otherwise assume it's an unboxed, pointerless value and do nothing. - def emit_dec_ref(self, dest: str, rtype: RType, is_xdec: bool = False) -> None: + def emit_dec_ref(self, + dest: str, + rtype: RType, + *, + is_xdec: bool = False, + rare: bool = False) -> None: """Decrement reference count of C expression `dest`. For composite unboxed structures (e.g. tuples) recursively decrement reference counts for each component. + + If rare is True, optimize for code size and compilation speed. """ x = 'X' if is_xdec else '' if is_int_rprimitive(rtype): - self.emit_line('CPyTagged_%sDecRef(%s);' % (x, dest)) + if rare: + self.emit_line('CPyTagged_%sDecRef(%s);' % (x, dest)) + else: + # Inlined + self.emit_line('CPyTagged_%sDECREF(%s);' % (x, dest)) elif isinstance(rtype, RTuple): for i, item_type in enumerate(rtype.types): - self.emit_dec_ref('{}.f{}'.format(dest, i), item_type, is_xdec) + self.emit_dec_ref('{}.f{}'.format(dest, i), item_type, is_xdec=is_xdec, rare=rare) elif not rtype.is_unboxed: - self.emit_line('CPy_%sDecRef(%s);' % (x, dest)) + if rare: + self.emit_line('CPy_%sDecRef(%s);' % (x, dest)) + else: + # Inlined + self.emit_line('CPy_%sDECREF(%s);' % (x, dest)) # Otherwise assume it's an unboxed, pointerless value and do nothing. def pretty_name(self, typ: RType) -> str: diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index 284ea094047ca..3b94c56487694 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -21,6 +21,7 @@ from mypyc.ir.func_ir import FuncIR, FuncDecl, FUNC_STATICMETHOD, FUNC_CLASSMETHOD, all_values from mypyc.ir.class_ir import ClassIR from mypyc.ir.pprint import generate_names_for_ir +from mypyc.analysis.blockfreq import frequently_executed_blocks # Whether to insert debug asserts for all error handling, to quickly # catch errors propagating without exceptions set. @@ -77,8 +78,11 @@ def generate_native_function(fn: FuncIR, for i, block in enumerate(blocks): block.label = i + common = frequently_executed_blocks(fn.blocks[0]) + for i in range(len(blocks)): block = blocks[i] + visitor.rare = block not in common next_block = None if i + 1 < len(blocks): next_block = blocks[i + 1] @@ -105,6 +109,7 @@ def __init__(self, self.source_path = source_path self.module_name = module_name self.literals = emitter.context.literals + self.rare = False self.next_block: Optional[BasicBlock] = None def temp_name(self) -> str: @@ -416,7 +421,7 @@ def visit_inc_ref(self, op: IncRef) -> None: def visit_dec_ref(self, op: DecRef) -> None: src = self.reg(op.src) - self.emit_dec_ref(src, op.src.type, op.is_xdec) + self.emit_dec_ref(src, op.src.type, is_xdec=op.is_xdec) def visit_box(self, op: Box) -> None: self.emitter.emit_box(self.reg(op.src), self.reg(op), op.src.type, can_borrow=True) @@ -574,10 +579,10 @@ def emit_lines(self, *lines: str) -> None: self.emitter.emit_lines(*lines) def emit_inc_ref(self, dest: str, rtype: RType) -> None: - self.emitter.emit_inc_ref(dest, rtype) + self.emitter.emit_inc_ref(dest, rtype, rare=self.rare) def emit_dec_ref(self, dest: str, rtype: RType, is_xdec: bool) -> None: - self.emitter.emit_dec_ref(dest, rtype, is_xdec) + self.emitter.emit_dec_ref(dest, rtype, is_xdec=is_xdec, rare=self.rare) def emit_declaration(self, line: str) -> None: self.declarations.emit_line(line) diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index 0915be50e9a9f..22dead2f79764 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -340,7 +340,8 @@ def __init__(self, self.negated = False # If not None, the true label should generate a traceback entry (func name, line number) self.traceback_entry: Optional[Tuple[str, int]] = None - # If True, the condition is expected to be usually False (for optimization purposes) + # If True, we expect to usually take the false branch (for optimization purposes); + # this is implicitly treated as true if there is a traceback entry self.rare = rare def targets(self) -> Sequence[BasicBlock]: diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 8aa7247bd66fd..987819154abf7 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -158,6 +158,24 @@ static inline int CPyTagged_CheckShort(CPyTagged x) { return !CPyTagged_CheckLong(x); } +static inline void CPyTagged_INCREF(CPyTagged x) { + if (unlikely(CPyTagged_CheckLong(x))) { + CPyTagged_IncRef(x); + } +} + +static inline void CPyTagged_DECREF(CPyTagged x) { + if (unlikely(CPyTagged_CheckLong(x))) { + CPyTagged_DecRef(x); + } +} + +static inline void CPyTagged_XDECREF(CPyTagged x) { + if (unlikely(CPyTagged_CheckLong(x))) { + CPyTagged_XDecRef(x); + } +} + static inline Py_ssize_t CPyTagged_ShortAsSsize_t(CPyTagged x) { // NOTE: Assume that we sign extend. return (Py_ssize_t)x >> 1; @@ -253,11 +271,10 @@ static inline bool CPyTagged_IsLe(CPyTagged left, CPyTagged right) { // Generic operations (that work with arbitrary types) -/* We use intentionally non-inlined decrefs since it pretty - * substantially speeds up compile time while only causing a ~1% - * performance degradation. We have our own copies both to avoid the - * null check in Py_DecRef and to avoid making an indirect PIC - * call. */ +/* We use intentionally non-inlined decrefs in rarely executed code + * paths since it pretty substantially speeds up compile time. We have + * our own copies both to avoid the null check in Py_DecRef and to avoid + * making an indirect PIC call. */ CPy_NOINLINE static void CPy_DecRef(PyObject *p) { CPy_DECREF(p); diff --git a/mypyc/test-data/exceptions-freq.test b/mypyc/test-data/exceptions-freq.test new file mode 100644 index 0000000000000..26b690a995031 --- /dev/null +++ b/mypyc/test-data/exceptions-freq.test @@ -0,0 +1,124 @@ +-- Test cases for basic block execution frequency analysis. +-- +-- These test cases are using exception transform test machinery for convenience. +-- +-- NOTE: These must all have the _freq suffix + +[case testSimpleError_freq] +from typing import List +def f(x: List[int]) -> int: + return x[0] +[out] +def f(x): + x :: list + r0 :: object + r1, r2 :: int +L0: + r0 = CPyList_GetItemShort(x, 0) + if is_error(r0) goto L3 (error at f:3) else goto L1 +L1: + r1 = unbox(int, r0) + dec_ref r0 + if is_error(r1) goto L3 (error at f:3) else goto L2 +L2: + return r1 +L3: + r2 = :: int + return r2 +hot blocks: [0, 1, 2] + +[case testHotBranch_freq] +from typing import List +def f(x: bool) -> None: + if x: + y = 1 + else: + y = 2 +[out] +def f(x): + x :: bool + y :: int +L0: + if x goto L1 else goto L2 :: bool +L1: + y = 2 + dec_ref y :: int + goto L3 +L2: + y = 4 + dec_ref y :: int +L3: + return 1 +hot blocks: [0, 1, 2, 3] + +[case testGoto_freq] +from typing import List +def f(x: bool) -> int: + if x: + y = 1 + else: + return 2 + return y +[out] +def f(x): + x :: bool + y :: int +L0: + if x goto L1 else goto L2 :: bool +L1: + y = 2 + goto L3 +L2: + return 4 +L3: + return y +hot blocks: [0, 1, 2, 3] + +[case testFalseOnError_freq] +from typing import List +def f(x: List[int]) -> None: + x[0] = 1 +[out] +def f(x): + x :: list + r0 :: object + r1 :: bit + r2 :: None +L0: + r0 = box(short_int, 2) + r1 = CPyList_SetItem(x, 0, r0) + if not r1 goto L2 (error at f:3) else goto L1 :: bool +L1: + return 1 +L2: + r2 = :: None + return r2 +hot blocks: [0, 1] + +[case testRareBranch_freq] +from typing_extensions import Final + +x: Final = str() + +def f() -> str: + return x +[out] +def f(): + r0 :: str + r1 :: bool + r2 :: str +L0: + r0 = __main__.x :: static + if is_error(r0) goto L1 else goto L3 +L1: + r1 = raise NameError('value for final name "x" was not set') + if not r1 goto L4 (error at f:6) else goto L2 :: bool +L2: + unreachable +L3: + inc_ref r0 + return r0 +L4: + r2 = :: str + return r2 +hot blocks: [0, 3] diff --git a/mypyc/test/test_emitfunc.py b/mypyc/test/test_emitfunc.py index a01524e9af780..a3f55a7facf2f 100644 --- a/mypyc/test/test_emitfunc.py +++ b/mypyc/test/test_emitfunc.py @@ -9,7 +9,7 @@ from mypyc.ir.ops import ( BasicBlock, Goto, Return, Integer, Assign, AssignMulti, IncRef, DecRef, Branch, Call, Unbox, Box, TupleGet, GetAttr, SetAttr, Op, Value, CallC, IntOp, LoadMem, - GetElementPtr, LoadAddress, ComparisonOp, SetMem, Register + GetElementPtr, LoadAddress, ComparisonOp, SetMem, Register, Unreachable ) from mypyc.ir.rtypes import ( RTuple, RInstance, RType, RArray, int_rprimitive, bool_rprimitive, list_rprimitive, @@ -202,18 +202,26 @@ def test_call_two_args(self) -> None: "cpy_r_r0 = CPyDef_myfn(cpy_r_m, cpy_r_k);") def test_inc_ref(self) -> None: - self.assert_emit(IncRef(self.m), - "CPyTagged_IncRef(cpy_r_m);") + self.assert_emit(IncRef(self.o), "CPy_INCREF(cpy_r_o);") + self.assert_emit(IncRef(self.o), "CPy_INCREF(cpy_r_o);", rare=True) def test_dec_ref(self) -> None: - self.assert_emit(DecRef(self.m), - "CPyTagged_DecRef(cpy_r_m);") + self.assert_emit(DecRef(self.o), "CPy_DECREF(cpy_r_o);") + self.assert_emit(DecRef(self.o), "CPy_DecRef(cpy_r_o);", rare=True) + + def test_inc_ref_int(self) -> None: + self.assert_emit(IncRef(self.m), "CPyTagged_INCREF(cpy_r_m);") + self.assert_emit(IncRef(self.m), "CPyTagged_IncRef(cpy_r_m);", rare=True) + + def test_dec_ref_int(self) -> None: + self.assert_emit(DecRef(self.m), "CPyTagged_DECREF(cpy_r_m);") + self.assert_emit(DecRef(self.m), "CPyTagged_DecRef(cpy_r_m);", rare=True) def test_dec_ref_tuple(self) -> None: - self.assert_emit(DecRef(self.t), 'CPyTagged_DecRef(cpy_r_t.f0);') + self.assert_emit(DecRef(self.t), 'CPyTagged_DECREF(cpy_r_t.f0);') def test_dec_ref_tuple_nested(self) -> None: - self.assert_emit(DecRef(self.tt), 'CPyTagged_DecRef(cpy_r_tt.f0.f0);') + self.assert_emit(DecRef(self.tt), 'CPyTagged_DECREF(cpy_r_tt.f0.f0);') def test_list_get_item(self) -> None: self.assert_emit(CallC(list_get_item_op.c_function_name, [self.m, self.k], @@ -253,7 +261,7 @@ def test_get_attr(self) -> None: if (unlikely(((mod___AObject *)cpy_r_r)->_y == CPY_INT_TAG)) { PyErr_SetString(PyExc_AttributeError, "attribute 'y' of 'A' undefined"); } else { - CPyTagged_IncRef(((mod___AObject *)cpy_r_r)->_y); + CPyTagged_INCREF(((mod___AObject *)cpy_r_r)->_y); } """) @@ -261,7 +269,7 @@ def test_set_attr(self) -> None: self.assert_emit( SetAttr(self.r, 'y', self.m, 1), """if (((mod___AObject *)cpy_r_r)->_y != CPY_INT_TAG) { - CPyTagged_DecRef(((mod___AObject *)cpy_r_r)->_y); + CPyTagged_DECREF(((mod___AObject *)cpy_r_r)->_y); } ((mod___AObject *)cpy_r_r)->_y = cpy_r_m; cpy_r_r0 = 1; @@ -383,7 +391,12 @@ def test_long_signed(self) -> None: self.assert_emit(Assign(a, Integer(-(1 << 31), int64_rprimitive)), """cpy_r_a = -2147483648LL;""") - def assert_emit(self, op: Op, expected: str, next_block: Optional[BasicBlock] = None) -> None: + def assert_emit(self, + op: Op, + expected: str, + next_block: Optional[BasicBlock] = None, + *, + rare: bool = False) -> None: block = BasicBlock(0) block.ops.append(op) value_names = generate_names_for_ir(self.registers, [block]) @@ -394,6 +407,7 @@ def assert_emit(self, op: Op, expected: str, next_block: Optional[BasicBlock] = visitor = FunctionEmitterVisitor(emitter, declarations, 'prog.py', 'prog') visitor.next_block = next_block + visitor.rare = rare op.accept(visitor) frags = declarations.fragments + emitter.fragments @@ -458,6 +472,7 @@ def test_register(self) -> None: reg = Register(int_rprimitive) op = Assign(reg, Integer(5)) self.block.ops.append(op) + self.block.ops.append(Unreachable()) fn = FuncIR(FuncDecl('myfunc', None, 'mod', FuncSignature([self.arg], list_rprimitive)), [self.reg], [self.block]) @@ -471,6 +486,7 @@ def test_register(self) -> None: ' CPyTagged cpy_r_r0;\n', 'CPyL0: ;\n', ' cpy_r_r0 = 10;\n', + ' CPy_Unreachable();\n', '}\n', ], result, msg='Generated code invalid') diff --git a/mypyc/test/test_exceptions.py b/mypyc/test/test_exceptions.py index 67df5ce5c38af..802024f2c86b1 100644 --- a/mypyc/test/test_exceptions.py +++ b/mypyc/test/test_exceptions.py @@ -18,9 +18,11 @@ ICODE_GEN_BUILTINS, use_custom_builtins, MypycDataSuite, build_ir_for_single_file, assert_test_output, remove_comment_lines ) +from mypyc.analysis.blockfreq import frequently_executed_blocks files = [ - 'exceptions.test' + 'exceptions.test', + 'exceptions-freq.test', ] @@ -46,6 +48,9 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: insert_exception_handling(fn) insert_ref_count_opcodes(fn) actual.extend(format_func(fn)) + if testcase.name.endswith('_freq'): + common = frequently_executed_blocks(fn.blocks[0]) + actual.append('hot blocks: %s' % sorted(b.label for b in common)) assert_test_output(testcase, actual, 'Invalid source code output', expected_output)