Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit c97d7e3

Browse files
JukkaLhauntsaninja
JukkaL
and
hauntsaninja
authored
[mypyc] Properly support native int argument defaults (#13577)
Since native ints can't have a dedicated reserved value to mark a missing argument, use extra bitmap arguments that are used to indicate whether default values should be used for particular arguments. The bitmap arguments are implicitly added at the end of a signature. They are not visible to user code and are just an implementation detail. This is analogous to how we track definedness of native int attributes using bitmaps. If a function accepts no arguments with a native int type and a default value, the bitmap arguments won't be generated. This doesn't handle inheritance properly. I'll fix inheritance in a follow-up PR. Work on mypyc/mypyc#837. Co-authored-by: Shantanu <[email protected]>
1 parent ad56164 commit c97d7e3

13 files changed

Lines changed: 577 additions & 55 deletions

File tree

‎mypyc/codegen/emit.py‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@
88

99
from mypyc.codegen.literals import Literals
1010
from mypyc.common import (
11-
ATTR_BITMAP_BITS,
1211
ATTR_PREFIX,
12+
BITMAP_BITS,
1313
FAST_ISINSTANCE_MAX_SUBCLASSES,
1414
NATIVE_PREFIX,
1515
REG_PREFIX,
@@ -332,7 +332,7 @@ def tuple_c_declaration(self, rtuple: RTuple) -> list[str]:
332332

333333
def bitmap_field(self, index: int) -> str:
334334
"""Return C field name used for attribute bitmap."""
335-
n = index // ATTR_BITMAP_BITS
335+
n = index // BITMAP_BITS
336336
if n == 0:
337337
return "bitmap"
338338
return f"bitmap{n + 1}"
@@ -366,7 +366,7 @@ def _emit_attr_bitmap_update(
366366
if value:
367367
self.emit_line(f"if (unlikely({value} == {self.c_undefined_value(rtype)})) {{")
368368
index = cl.bitmap_attrs.index(attr)
369-
mask = 1 << (index & (ATTR_BITMAP_BITS - 1))
369+
mask = 1 << (index & (BITMAP_BITS - 1))
370370
bitmap = self.attr_bitmap_expr(obj, cl, index)
371371
if clear:
372372
self.emit_line(f"{bitmap} &= ~{mask};")
@@ -400,7 +400,7 @@ def emit_undefined_attr_check(
400400
check = f"unlikely({check})"
401401
if is_fixed_width_rtype(rtype):
402402
index = cl.bitmap_attrs.index(attr)
403-
bit = 1 << (index & (ATTR_BITMAP_BITS - 1))
403+
bit = 1 << (index & (BITMAP_BITS - 1))
404404
attr = self.bitmap_field(index)
405405
obj_expr = f"({cl.struct_name(self.names)} *){obj}"
406406
check = f"{check} && !(({obj_expr})->{attr} & {bit})"
@@ -986,7 +986,11 @@ def emit_box(
986986

987987
def emit_error_check(self, value: str, rtype: RType, failure: str) -> None:
988988
"""Emit code for checking a native function return value for uncaught exception."""
989-
if not isinstance(rtype, RTuple):
989+
if is_fixed_width_rtype(rtype):
990+
# The error value is also valid as a normal value, so we need to also check
991+
# for a raised exception.
992+
self.emit_line(f"if ({value} == {self.c_error_value(rtype)} && PyErr_Occurred()) {{")
993+
elif not isinstance(rtype, RTuple):
990994
self.emit_line(f"if ({value} == {self.c_error_value(rtype)}) {{")
991995
else:
992996
if len(rtype.types) == 0:

‎mypyc/codegen/emitclass.py‎

Lines changed: 4 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,7 @@
1717
generate_richcompare_wrapper,
1818
generate_set_del_item_wrapper,
1919
)
20-
from mypyc.common import (
21-
ATTR_BITMAP_BITS,
22-
ATTR_BITMAP_TYPE,
23-
NATIVE_PREFIX,
24-
PREFIX,
25-
REG_PREFIX,
26-
use_fastcall,
27-
)
20+
from mypyc.common import BITMAP_BITS, BITMAP_TYPE, NATIVE_PREFIX, PREFIX, REG_PREFIX, use_fastcall
2821
from mypyc.ir.class_ir import ClassIR, VTableEntries
2922
from mypyc.ir.func_ir import FUNC_CLASSMETHOD, FUNC_STATICMETHOD, FuncDecl, FuncIR
3023
from mypyc.ir.rtypes import RTuple, RType, is_fixed_width_rtype, object_rprimitive
@@ -385,10 +378,10 @@ def generate_object_struct(cl: ClassIR, emitter: Emitter) -> None:
385378
if base.bitmap_attrs:
386379
# Do we need another attribute bitmap field?
387380
if emitter.bitmap_field(len(base.bitmap_attrs) - 1) not in bitmap_attrs:
388-
for i in range(0, len(base.bitmap_attrs), ATTR_BITMAP_BITS):
381+
for i in range(0, len(base.bitmap_attrs), BITMAP_BITS):
389382
attr = emitter.bitmap_field(i)
390383
if attr not in bitmap_attrs:
391-
lines.append(f"{ATTR_BITMAP_TYPE} {attr};")
384+
lines.append(f"{BITMAP_TYPE} {attr};")
392385
bitmap_attrs.append(attr)
393386
for attr, rtype in base.attributes.items():
394387
if (attr, rtype) not in seen_attrs:
@@ -567,7 +560,7 @@ def generate_setup_for_class(
567560
emitter.emit_line("}")
568561
else:
569562
emitter.emit_line(f"self->vtable = {vtable_name};")
570-
for i in range(0, len(cl.bitmap_attrs), ATTR_BITMAP_BITS):
563+
for i in range(0, len(cl.bitmap_attrs), BITMAP_BITS):
571564
field = emitter.bitmap_field(i)
572565
emitter.emit_line(f"self->{field} = 0;")
573566

‎mypyc/codegen/emitwrapper.py‎

Lines changed: 69 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,22 @@
1717
from mypy.nodes import ARG_NAMED, ARG_NAMED_OPT, ARG_OPT, ARG_POS, ARG_STAR, ARG_STAR2, ArgKind
1818
from mypy.operators import op_methods_to_symbols, reverse_op_method_names, reverse_op_methods
1919
from mypyc.codegen.emit import AssignHandler, Emitter, ErrorHandler, GotoHandler, ReturnHandler
20-
from mypyc.common import DUNDER_PREFIX, NATIVE_PREFIX, PREFIX, use_vectorcall
20+
from mypyc.common import (
21+
BITMAP_BITS,
22+
BITMAP_TYPE,
23+
DUNDER_PREFIX,
24+
NATIVE_PREFIX,
25+
PREFIX,
26+
bitmap_name,
27+
use_vectorcall,
28+
)
2129
from mypyc.ir.class_ir import ClassIR
2230
from mypyc.ir.func_ir import FUNC_STATICMETHOD, FuncIR, RuntimeArg
2331
from mypyc.ir.rtypes import (
2432
RInstance,
2533
RType,
2634
is_bool_rprimitive,
35+
is_fixed_width_rtype,
2736
is_int_rprimitive,
2837
is_object_rprimitive,
2938
object_rprimitive,
@@ -135,6 +144,8 @@ def generate_wrapper_function(
135144

136145
# If fn is a method, then the first argument is a self param
137146
real_args = list(fn.args)
147+
if fn.sig.num_bitmap_args:
148+
real_args = real_args[: -fn.sig.num_bitmap_args]
138149
if fn.class_name and not fn.decl.kind == FUNC_STATICMETHOD:
139150
arg = real_args.pop(0)
140151
emitter.emit_line(f"PyObject *obj_{arg.name} = self;")
@@ -185,6 +196,9 @@ def generate_wrapper_function(
185196
"return NULL;",
186197
"}",
187198
)
199+
for i in range(fn.sig.num_bitmap_args):
200+
name = bitmap_name(i)
201+
emitter.emit_line(f"{BITMAP_TYPE} {name} = 0;")
188202
traceback_code = generate_traceback_code(fn, emitter, source_path, module_name)
189203
generate_wrapper_core(
190204
fn,
@@ -223,6 +237,8 @@ def generate_legacy_wrapper_function(
223237

224238
# If fn is a method, then the first argument is a self param
225239
real_args = list(fn.args)
240+
if fn.sig.num_bitmap_args:
241+
real_args = real_args[: -fn.sig.num_bitmap_args]
226242
if fn.class_name and not fn.decl.kind == FUNC_STATICMETHOD:
227243
arg = real_args.pop(0)
228244
emitter.emit_line(f"PyObject *obj_{arg.name} = self;")
@@ -254,6 +270,9 @@ def generate_legacy_wrapper_function(
254270
"return NULL;",
255271
"}",
256272
)
273+
for i in range(fn.sig.num_bitmap_args):
274+
name = bitmap_name(i)
275+
emitter.emit_line(f"{BITMAP_TYPE} {name} = 0;")
257276
traceback_code = generate_traceback_code(fn, emitter, source_path, module_name)
258277
generate_wrapper_core(
259278
fn,
@@ -669,7 +688,8 @@ def generate_wrapper_core(
669688
"""
670689
gen = WrapperGenerator(None, emitter)
671690
gen.set_target(fn)
672-
gen.arg_names = arg_names or [arg.name for arg in fn.args]
691+
if arg_names:
692+
gen.arg_names = arg_names
673693
gen.cleanups = cleanups or []
674694
gen.optional_args = optional_args or []
675695
gen.traceback_code = traceback_code or ""
@@ -688,6 +708,7 @@ def generate_arg_check(
688708
*,
689709
optional: bool = False,
690710
raise_exception: bool = True,
711+
bitmap_arg_index: int = 0,
691712
) -> None:
692713
"""Insert a runtime check for argument and unbox if necessary.
693714
@@ -697,17 +718,34 @@ def generate_arg_check(
697718
"""
698719
error = error or AssignHandler()
699720
if typ.is_unboxed:
700-
# Borrow when unboxing to avoid reference count manipulation.
701-
emitter.emit_unbox(
702-
f"obj_{name}",
703-
f"arg_{name}",
704-
typ,
705-
declare_dest=True,
706-
raise_exception=raise_exception,
707-
error=error,
708-
borrow=True,
709-
optional=optional,
710-
)
721+
if is_fixed_width_rtype(typ) and optional:
722+
# Update bitmap is value is provided.
723+
emitter.emit_line(f"{emitter.ctype(typ)} arg_{name} = 0;")
724+
emitter.emit_line(f"if (obj_{name} != NULL) {{")
725+
bitmap = bitmap_name(bitmap_arg_index // BITMAP_BITS)
726+
emitter.emit_line(f"{bitmap} |= 1 << {bitmap_arg_index & (BITMAP_BITS - 1)};")
727+
emitter.emit_unbox(
728+
f"obj_{name}",
729+
f"arg_{name}",
730+
typ,
731+
declare_dest=False,
732+
raise_exception=raise_exception,
733+
error=error,
734+
borrow=True,
735+
)
736+
emitter.emit_line("}")
737+
else:
738+
# Borrow when unboxing to avoid reference count manipulation.
739+
emitter.emit_unbox(
740+
f"obj_{name}",
741+
f"arg_{name}",
742+
typ,
743+
declare_dest=True,
744+
raise_exception=raise_exception,
745+
error=error,
746+
borrow=True,
747+
optional=optional,
748+
)
711749
elif is_object_rprimitive(typ):
712750
# Object is trivial since any object is valid
713751
if optional:
@@ -749,8 +787,12 @@ def set_target(self, fn: FuncIR) -> None:
749787
"""
750788
self.target_name = fn.name
751789
self.target_cname = fn.cname(self.emitter.names)
752-
self.arg_names = [arg.name for arg in fn.args]
753-
self.args = fn.args[:]
790+
self.num_bitmap_args = fn.sig.num_bitmap_args
791+
if self.num_bitmap_args:
792+
self.args = fn.args[: -self.num_bitmap_args]
793+
else:
794+
self.args = fn.args
795+
self.arg_names = [arg.name for arg in self.args]
754796
self.ret_type = fn.ret_type
755797

756798
def wrapper_name(self) -> str:
@@ -779,17 +821,22 @@ def emit_arg_processing(
779821
) -> None:
780822
"""Emit validation and unboxing of arguments."""
781823
error = error or self.error()
824+
bitmap_arg_index = 0
782825
for arg_name, arg in zip(self.arg_names, self.args):
783826
# Suppress the argument check for *args/**kwargs, since we know it must be right.
784827
typ = arg.type if arg.kind not in (ARG_STAR, ARG_STAR2) else object_rprimitive
828+
optional = arg in self.optional_args
785829
generate_arg_check(
786830
arg_name,
787831
typ,
788832
self.emitter,
789833
error,
790834
raise_exception=raise_exception,
791-
optional=arg in self.optional_args,
835+
optional=optional,
836+
bitmap_arg_index=bitmap_arg_index,
792837
)
838+
if optional and is_fixed_width_rtype(typ):
839+
bitmap_arg_index += 1
793840

794841
def emit_call(self, not_implemented_handler: str = "") -> None:
795842
"""Emit call to the wrapper function.
@@ -798,6 +845,12 @@ def emit_call(self, not_implemented_handler: str = "") -> None:
798845
a NotImplemented return value (if it's possible based on the return type).
799846
"""
800847
native_args = ", ".join(f"arg_{arg}" for arg in self.arg_names)
848+
if self.num_bitmap_args:
849+
bitmap_args = ", ".join(
850+
[bitmap_name(i) for i in reversed(range(self.num_bitmap_args))]
851+
)
852+
native_args = f"{native_args}, {bitmap_args}"
853+
801854
ret_type = self.ret_type
802855
emitter = self.emitter
803856
if ret_type.is_unboxed or self.use_goto():

‎mypyc/common.py‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -53,10 +53,12 @@
5353
MAX_LITERAL_SHORT_INT: Final = sys.maxsize >> 1 if not IS_MIXED_32_64_BIT_BUILD else 2**30 - 1
5454
MIN_LITERAL_SHORT_INT: Final = -MAX_LITERAL_SHORT_INT - 1
5555

56-
# Decription of the C type used to track definedness of attributes
57-
# that have types with overlapping error values
58-
ATTR_BITMAP_TYPE: Final = "uint32_t"
59-
ATTR_BITMAP_BITS: Final = 32
56+
# Decription of the C type used to track the definedness of attributes and
57+
# the presence of argument default values that have types with overlapping
58+
# error values. Each tracked attribute/argument has a dedicated bit in the
59+
# relevant bitmap.
60+
BITMAP_TYPE: Final = "uint32_t"
61+
BITMAP_BITS: Final = 32
6062

6163
# Runtime C library files
6264
RUNTIME_C_FILES: Final = [
@@ -128,3 +130,9 @@ def short_id_from_name(func_name: str, shortname: str, line: int | None) -> str:
128130
else:
129131
partial_name = shortname
130132
return partial_name
133+
134+
135+
def bitmap_name(index: int) -> str:
136+
if index == 0:
137+
return "__bitmap"
138+
return f"__bitmap{index + 1}"

‎mypyc/ir/func_ir.py‎

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
from typing_extensions import Final
77

88
from mypy.nodes import ARG_POS, ArgKind, Block, FuncDef
9-
from mypyc.common import JsonDict, get_id_from_name, short_id_from_name
9+
from mypyc.common import BITMAP_BITS, JsonDict, bitmap_name, get_id_from_name, short_id_from_name
1010
from mypyc.ir.ops import (
1111
Assign,
1212
AssignMulti,
@@ -17,7 +17,7 @@
1717
Register,
1818
Value,
1919
)
20-
from mypyc.ir.rtypes import RType, deserialize_type
20+
from mypyc.ir.rtypes import RType, bitmap_rprimitive, deserialize_type, is_fixed_width_rtype
2121
from mypyc.namegen import NameGenerator
2222

2323

@@ -70,12 +70,29 @@ class FuncSignature:
7070
def __init__(self, args: Sequence[RuntimeArg], ret_type: RType) -> None:
7171
self.args = tuple(args)
7272
self.ret_type = ret_type
73+
self.num_bitmap_args = num_bitmap_args(self.args)
74+
if self.num_bitmap_args:
75+
extra = [
76+
RuntimeArg(bitmap_name(i), bitmap_rprimitive, pos_only=True)
77+
for i in range(self.num_bitmap_args)
78+
]
79+
self.args = self.args + tuple(reversed(extra))
80+
81+
def bound_sig(self) -> "FuncSignature":
82+
if self.num_bitmap_args:
83+
return FuncSignature(self.args[1 : -self.num_bitmap_args], self.ret_type)
84+
else:
85+
return FuncSignature(self.args[1:], self.ret_type)
7386

7487
def __repr__(self) -> str:
7588
return f"FuncSignature(args={self.args!r}, ret={self.ret_type!r})"
7689

7790
def serialize(self) -> JsonDict:
78-
return {"args": [t.serialize() for t in self.args], "ret_type": self.ret_type.serialize()}
91+
if self.num_bitmap_args:
92+
args = self.args[: -self.num_bitmap_args]
93+
else:
94+
args = self.args
95+
return {"args": [t.serialize() for t in args], "ret_type": self.ret_type.serialize()}
7996

8097
@classmethod
8198
def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> FuncSignature:
@@ -85,6 +102,14 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> FuncSignature:
85102
)
86103

87104

105+
def num_bitmap_args(args: tuple[RuntimeArg, ...]) -> int:
106+
n = 0
107+
for arg in args:
108+
if is_fixed_width_rtype(arg.type) and arg.kind.is_optional():
109+
n += 1
110+
return (n + (BITMAP_BITS - 1)) // BITMAP_BITS
111+
112+
88113
FUNC_NORMAL: Final = 0
89114
FUNC_STATICMETHOD: Final = 1
90115
FUNC_CLASSMETHOD: Final = 2
@@ -120,7 +145,7 @@ def __init__(
120145
if kind == FUNC_STATICMETHOD:
121146
self.bound_sig = sig
122147
else:
123-
self.bound_sig = FuncSignature(sig.args[1:], sig.ret_type)
148+
self.bound_sig = sig.bound_sig()
124149

125150
# this is optional because this will be set to the line number when the corresponding
126151
# FuncIR is created

‎mypyc/ir/rtypes.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -361,6 +361,9 @@ def __hash__(self) -> int:
361361
"c_ptr", is_unboxed=False, is_refcounted=False, ctype="void *"
362362
)
363363

364+
# The type corresponding to mypyc.common.BITMAP_TYPE
365+
bitmap_rprimitive: Final = uint32_rprimitive
366+
364367
# Floats are represent as 'float' PyObject * values. (In the future
365368
# we'll likely switch to a more efficient, unboxed representation.)
366369
float_rprimitive: Final = RPrimitive("builtins.float", is_unboxed=False, is_refcounted=True)

0 commit comments

Comments
 (0)