From b71773c44da16427aef8b3160b41dd503eb15bb7 Mon Sep 17 00:00:00 2001 From: Advait Dixit Date: Thu, 23 Jan 2025 09:43:12 -0800 Subject: [PATCH 1/9] [mypyc] Calling tp_finalize from tp_dealloc. --- mypyc/codegen/emitclass.py | 27 +++++++++++++++++++++++++++ mypyc/test-data/run-classes.test | 20 ++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 54c979482f663..8b82361eabb73 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -205,6 +205,7 @@ def generate_class(cl: ClassIR, module: str, emitter: Emitter) -> None: setup_name = f"{name_prefix}_setup" new_name = f"{name_prefix}_new" + finalize_name = f"{name_prefix}_finalize" members_name = f"{name_prefix}_members" getseters_name = f"{name_prefix}_getseters" vtable_name = f"{name_prefix}_vtable" @@ -222,10 +223,14 @@ def generate_class(cl: ClassIR, module: str, emitter: Emitter) -> None: if not cl.builtin_base: fields["tp_new"] = new_name + # Populate .tp_finalize and generate a finalize method only if __del__ is defined for this class. + del_method = next((e.method for e in cl.vtable_entries if e.name == "__del__"), None) if generate_full: fields["tp_dealloc"] = f"(destructor){name_prefix}_dealloc" fields["tp_traverse"] = f"(traverseproc){name_prefix}_traverse" fields["tp_clear"] = f"(inquiry){name_prefix}_clear" + if del_method: + fields["tp_finalize"] = f"(destructor){finalize_name}" if needs_getseters: fields["tp_getset"] = getseters_name fields["tp_methods"] = methods_name @@ -308,6 +313,9 @@ def emit_line() -> None: emit_line() generate_dealloc_for_class(cl, dealloc_name, clear_name, emitter) emit_line() + if del_method: + generate_finalize_for_class(del_method, finalize_name, emitter) + emit_line() if cl.allow_interpreted_subclasses: shadow_vtable_name: str | None = generate_vtables( @@ -779,6 +787,9 @@ def generate_dealloc_for_class( emitter.emit_line("static void") emitter.emit_line(f"{dealloc_func_name}({cl.struct_name(emitter.names)} *self)") emitter.emit_line("{") + emitter.emit_line("if (Py_TYPE(self)->tp_finalize) {") + emitter.emit_line(" Py_TYPE(self)->tp_finalize((PyObject *)self);") + emitter.emit_line("}") emitter.emit_line("PyObject_GC_UnTrack(self);") # The trashcan is needed to handle deep recursive deallocations emitter.emit_line(f"CPy_TRASHCAN_BEGIN(self, {dealloc_func_name})") @@ -788,6 +799,22 @@ def generate_dealloc_for_class( emitter.emit_line("}") +def generate_finalize_for_class( + del_method: FuncIR, finalize_func_name: str, emitter: Emitter +) -> None: + emitter.emit_line("static void") + emitter.emit_line(f"{finalize_func_name}(PyObject *self)") + emitter.emit_line("{") + emitter.emit_line( + "{}{}{}(self);".format( + emitter.get_group_prefix(del_method.decl), + NATIVE_PREFIX, + del_method.cname(emitter.names), + ) + ) + emitter.emit_line("}") + + def generate_methods_table(cl: ClassIR, name: str, emitter: Emitter) -> None: emitter.emit_line(f"static PyMethodDef {name}[] = {{") for fn in cl.methods.values(): diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 168477d5a8eec..cb6e6395a7aad 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2719,3 +2719,23 @@ print(native.A(ints=[1, -17]).ints) [out] \[1, -17] + +[case testDel] +class A: + def __del__(self): + print("deleting A...") + +class B(A): + def __init__(self): + self.a = A() + + def __del__(self): + print("deleting B...") + +[file driver.py] +import native +native.B() + +[out] +deleting B... +deleting A... From 4e9e6087b95a593a782711fc066f087b91f33987 Mon Sep 17 00:00:00 2001 From: Advait Dixit Date: Thu, 23 Jan 2025 10:55:15 -0800 Subject: [PATCH 2/9] Moving code for readability. --- mypyc/codegen/emitclass.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 8b82361eabb73..efdd20f165fcd 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -223,12 +223,12 @@ def generate_class(cl: ClassIR, module: str, emitter: Emitter) -> None: if not cl.builtin_base: fields["tp_new"] = new_name - # Populate .tp_finalize and generate a finalize method only if __del__ is defined for this class. - del_method = next((e.method for e in cl.vtable_entries if e.name == "__del__"), None) if generate_full: fields["tp_dealloc"] = f"(destructor){name_prefix}_dealloc" fields["tp_traverse"] = f"(traverseproc){name_prefix}_traverse" fields["tp_clear"] = f"(inquiry){name_prefix}_clear" + # Populate .tp_finalize and generate a finalize method only if __del__ is defined for this class. + del_method = next((e.method for e in cl.vtable_entries if e.name == "__del__"), None) if del_method: fields["tp_finalize"] = f"(destructor){finalize_name}" if needs_getseters: From 868038a79b3c95f97d6eff74a981e1bc24600ff1 Mon Sep 17 00:00:00 2001 From: Advait Dixit Date: Thu, 23 Jan 2025 11:32:19 -0800 Subject: [PATCH 3/9] Enhancing test. --- mypyc/test-data/run-classes.test | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index cb6e6395a7aad..5daf8c7d5b37b 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2725,17 +2725,23 @@ class A: def __del__(self): print("deleting A...") -class B(A): +class B: + def __del__(self): + print("deleting B...") + +class C(B): def __init__(self): self.a = A() def __del__(self): - print("deleting B...") + print("deleting C...") + super().__del__() [file driver.py] import native -native.B() +native.C() [out] +deleting C... deleting B... deleting A... From 36cc4948086827392ce01ccc8c208c04a64fef52 Mon Sep 17 00:00:00 2001 From: Advait Dixit Date: Wed, 29 Jan 2025 15:30:42 +1100 Subject: [PATCH 4/9] * Moving null-check of tp_finalize from runtime to compile time. * Calling PyObject_GC_IsFinalized(...) to ensure tp_finalize is called exactly once. --- mypyc/codegen/emitclass.py | 16 +++++++++----- mypyc/test-data/run-classes.test | 36 ++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index efdd20f165fcd..addacac3f69eb 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -311,7 +311,7 @@ def emit_line() -> None: emit_line() generate_clear_for_class(cl, clear_name, emitter) emit_line() - generate_dealloc_for_class(cl, dealloc_name, clear_name, emitter) + generate_dealloc_for_class(cl, dealloc_name, clear_name, bool(del_method), emitter) emit_line() if del_method: generate_finalize_for_class(del_method, finalize_name, emitter) @@ -782,14 +782,15 @@ def generate_clear_for_class(cl: ClassIR, func_name: str, emitter: Emitter) -> N def generate_dealloc_for_class( - cl: ClassIR, dealloc_func_name: str, clear_func_name: str, emitter: Emitter + cl: ClassIR, dealloc_func_name: str, clear_func_name: str, has_tp_finalize: bool, emitter: Emitter ) -> None: emitter.emit_line("static void") emitter.emit_line(f"{dealloc_func_name}({cl.struct_name(emitter.names)} *self)") emitter.emit_line("{") - emitter.emit_line("if (Py_TYPE(self)->tp_finalize) {") - emitter.emit_line(" Py_TYPE(self)->tp_finalize((PyObject *)self);") - emitter.emit_line("}") + if has_tp_finalize: + emitter.emit_line("if (!PyObject_GC_IsFinalized((PyObject *)self)) {") + emitter.emit_line("Py_TYPE(self)->tp_finalize((PyObject *)self);") + emitter.emit_line("}") emitter.emit_line("PyObject_GC_UnTrack(self);") # The trashcan is needed to handle deep recursive deallocations emitter.emit_line(f"CPy_TRASHCAN_BEGIN(self, {dealloc_func_name})") @@ -805,6 +806,8 @@ def generate_finalize_for_class( emitter.emit_line("static void") emitter.emit_line(f"{finalize_func_name}(PyObject *self)") emitter.emit_line("{") + emitter.emit_line("PyObject *type, *value, *traceback;") + emitter.emit_line("PyErr_Fetch(&type, &value, &traceback);") emitter.emit_line( "{}{}{}(self);".format( emitter.get_group_prefix(del_method.decl), @@ -812,6 +815,9 @@ def generate_finalize_for_class( del_method.cname(emitter.names), ) ) + emitter.emit_line("if (type != NULL) {") + emitter.emit_line("PyErr_Restore(type, value, traceback);") + emitter.emit_line("}") emitter.emit_line("}") diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 5daf8c7d5b37b..a0614d677bfd8 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2737,11 +2737,47 @@ class C(B): print("deleting C...") super().__del__() +class D(A): + pass + [file driver.py] import native native.C() +native.D() [out] deleting C... deleting B... deleting A... +deleting A... + +[case testDelCircular] +import dataclasses +import typing + +i: int = 1 + +@dataclasses.dataclass +class C: + var: typing.Optional["C"] = dataclasses.field(default=None) + + def __del__(self): + global i + print(f"deleting C{i}...") + i = i + 1 + +[file driver.py] +import native +import gc + +c1 = native.C() +c2 = native.C() +c1.var = c2 +c2.var = c1 +del c1 +del c2 +gc.collect() + +[out] +deleting C1... +deleting C2... From 6cf1116973fe40a78dc400e8c0fc7c1c3d6580a5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 29 Jan 2025 04:34:37 +0000 Subject: [PATCH 5/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mypyc/codegen/emitclass.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index addacac3f69eb..6b79d5e6539bd 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -782,7 +782,11 @@ def generate_clear_for_class(cl: ClassIR, func_name: str, emitter: Emitter) -> N def generate_dealloc_for_class( - cl: ClassIR, dealloc_func_name: str, clear_func_name: str, has_tp_finalize: bool, emitter: Emitter + cl: ClassIR, + dealloc_func_name: str, + clear_func_name: str, + has_tp_finalize: bool, + emitter: Emitter, ) -> None: emitter.emit_line("static void") emitter.emit_line(f"{dealloc_func_name}({cl.struct_name(emitter.names)} *self)") From 097efaf88d21bdd83ec428e2f7b83336a3f5f189 Mon Sep 17 00:00:00 2001 From: Advait Dixit Date: Wed, 29 Jan 2025 21:14:02 +1100 Subject: [PATCH 6/9] Fixed handling of exceptions raised in __del__ methods. --- mypyc/codegen/emitclass.py | 14 ++++++++++++-- mypyc/test-data/run-classes.test | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 6b79d5e6539bd..8a58c8f08c721 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -819,9 +819,19 @@ def generate_finalize_for_class( del_method.cname(emitter.names), ) ) - emitter.emit_line("if (type != NULL) {") - emitter.emit_line("PyErr_Restore(type, value, traceback);") + emitter.emit_line("if (PyErr_Occurred() != NULL) {") + emitter.emit_line('PyObject *class_obj = PyObject_GetAttrString(self, "__class__");') + emitter.emit_line('PyObject *del_method = PyObject_GetAttrString(class_obj, "__del__");') + # CPython interpreter uses PyErr_WriteUnraisable: https://docs.python.org/3/c-api/exceptions.html#c.PyErr_WriteUnraisable + # However, the message is slightly different due to the way mypyc compiles classes. + # CPython interpreter prints: Exception ignored in: + # mypyc prints: Exception ignored in: + emitter.emit_line("PyErr_WriteUnraisable(del_method);") + emitter.emit_line("Py_DECREF(class_obj);") + emitter.emit_line("Py_XDECREF(del_method);") emitter.emit_line("}") + # PyErr_Restore also clears exception raised in __del__. + emitter.emit_line("PyErr_Restore(type, value, traceback);") emitter.emit_line("}") diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index a0614d677bfd8..c727ec0575987 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2781,3 +2781,22 @@ gc.collect() [out] deleting C1... deleting C2... + +[case testDelException] +# The error message in the expected output of this test does not match CPython's error message due to the way mypyc compiles Python classes. If the error message is fixed, the expected output of this test will also change. +class F: + def __del__(self): + if True: + raise Exception("e2") + +[file driver.py] +import native +f = native.F() +del f + +[out] +Exception ignored in: +Traceback (most recent call last): + File "native.py", line 5, in __del__ + raise Exception("e2") +Exception: e2 From d4e5f403d524fe66ab326124ac02c1438ba2cc57 Mon Sep 17 00:00:00 2001 From: Advait Dixit Date: Sat, 8 Feb 2025 07:48:56 +1100 Subject: [PATCH 7/9] Fixing ignored exception message. --- mypyc/codegen/emitclass.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 8a58c8f08c721..9839be49ccf64 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -820,15 +820,15 @@ def generate_finalize_for_class( ) ) emitter.emit_line("if (PyErr_Occurred() != NULL) {") - emitter.emit_line('PyObject *class_obj = PyObject_GetAttrString(self, "__class__");') - emitter.emit_line('PyObject *del_method = PyObject_GetAttrString(class_obj, "__del__");') + emitter.emit_line('PyObject *del_str = PyUnicode_FromString("__del__");') + emitter.emit_line("PyObject *del_method = _PyType_Lookup(Py_TYPE(self), del_str);") # CPython interpreter uses PyErr_WriteUnraisable: https://docs.python.org/3/c-api/exceptions.html#c.PyErr_WriteUnraisable # However, the message is slightly different due to the way mypyc compiles classes. # CPython interpreter prints: Exception ignored in: # mypyc prints: Exception ignored in: emitter.emit_line("PyErr_WriteUnraisable(del_method);") - emitter.emit_line("Py_DECREF(class_obj);") emitter.emit_line("Py_XDECREF(del_method);") + emitter.emit_line("Py_XDECREF(del_str);") emitter.emit_line("}") # PyErr_Restore also clears exception raised in __del__. emitter.emit_line("PyErr_Restore(type, value, traceback);") From 346f415858e705b7e73a02ca843388811bc456a9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 8 Feb 2025 07:07:37 +0000 Subject: [PATCH 8/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mypyc/test-data/run-classes.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 48bc088051a70..22c7362d1ef09 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2828,4 +2828,4 @@ Exception ignored in: Traceback (most recent call last): File "native.py", line 5, in __del__ raise Exception("e2") -Exception: e2 \ No newline at end of file +Exception: e2 From c942ddb78b3d5de2c2f757049d17618c3b88130d Mon Sep 17 00:00:00 2001 From: Advait Dixit Date: Thu, 13 Feb 2025 14:26:43 -0800 Subject: [PATCH 9/9] Adding NULL check for del_str. --- mypyc/codegen/emitclass.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index d03f59a6b295f..c5191e5fb9397 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -812,7 +812,9 @@ def generate_finalize_for_class( ) emitter.emit_line("if (PyErr_Occurred() != NULL) {") emitter.emit_line('PyObject *del_str = PyUnicode_FromString("__del__");') - emitter.emit_line("PyObject *del_method = _PyType_Lookup(Py_TYPE(self), del_str);") + emitter.emit_line( + "PyObject *del_method = (del_str == NULL) ? NULL : _PyType_Lookup(Py_TYPE(self), del_str);" + ) # CPython interpreter uses PyErr_WriteUnraisable: https://docs.python.org/3/c-api/exceptions.html#c.PyErr_WriteUnraisable # However, the message is slightly different due to the way mypyc compiles classes. # CPython interpreter prints: Exception ignored in: