From 1d3f8145cae1270f07c8adcbb14861ba5ccef3c8 Mon Sep 17 00:00:00 2001 From: Jingchen Ye <97littleleaf11@gmail.com> Date: Sun, 25 Jul 2021 19:27:45 +0800 Subject: [PATCH 1/6] Fix %c string and bytes interpolation --- mypy/checkstrformat.py | 23 ++++++++++++------ test-data/unit/check-expressions.test | 35 ++++++++++++++++++++++++++- test-data/unit/fixtures/python2.pyi | 2 ++ 3 files changed, 51 insertions(+), 9 deletions(-) diff --git a/mypy/checkstrformat.py b/mypy/checkstrformat.py index 78628dc34ba06..e9d3f96002b24 100644 --- a/mypy/checkstrformat.py +++ b/mypy/checkstrformat.py @@ -786,7 +786,7 @@ def build_replacement_checkers(self, specifiers: List[ConversionSpecifier], def replacement_checkers(self, specifier: ConversionSpecifier, context: Context, expr: FormatStringExpr) -> Optional[List[Checkers]]: """Returns a list of tuples of two functions that check whether a replacement is - of the right type for the specifier. The first functions take a node and checks + of the right type for the specifier. The first function takes a node and checks its type in the right type context. The second function just checks a type. """ checkers: List[Checkers] = [] @@ -874,11 +874,11 @@ def check_s_special_cases(self, expr: FormatStringExpr, typ: Type, context: Cont def checkers_for_c_type(self, type: str, context: Context, - expr: FormatStringExpr) -> Optional[Checkers]: + format_expr: FormatStringExpr) -> Optional[Checkers]: """Returns a tuple of check functions that check whether, respectively, a node or a type is compatible with 'type' that is a character type. """ - expected_type = self.conversion_type(type, context, expr) + expected_type = self.conversion_type(type, context, format_expr) if expected_type is None: return None @@ -889,8 +889,12 @@ def check_type(type: Type) -> None: def check_expr(expr: Expression) -> None: """int, or str with length 1""" type = self.accept(expr, expected_type) - if isinstance(expr, (StrExpr, BytesExpr)) and len(cast(StrExpr, expr).value) != 1: - self.msg.requires_int_or_char(context) + # TODO: Use the same the error message when incompatible types match %c + # Python 3 doesn't support b'%c' % str + if not (self.chk.options.python_version >= (3, 0) + and isinstance(format_expr, BytesExpr)): + if isinstance(expr, (StrExpr, BytesExpr)) and len(cast(StrExpr, expr).value) != 1: + self.msg.requires_int_or_char(context) check_type(type) return check_expr, check_type @@ -939,9 +943,12 @@ def conversion_type(self, p: str, context: Context, expr: FormatStringExpr, numeric_types.append(self.named_type('typing.SupportsInt')) return UnionType.make_union(numeric_types) elif p in ['c']: - return UnionType([self.named_type('builtins.int'), - self.named_type('builtins.float'), - self.named_type('builtins.str')]) + if isinstance(expr, BytesExpr): + return UnionType([self.named_type('builtins.int'), + self.named_type('builtins.bytes')]) + else: + return UnionType([self.named_type('builtins.int'), + self.named_type('builtins.str')]) else: self.msg.unsupported_placeholder(p, context) return None diff --git a/test-data/unit/check-expressions.test b/test-data/unit/check-expressions.test index 5d3d6b66d7b8a..84727aea8336e 100644 --- a/test-data/unit/check-expressions.test +++ b/test-data/unit/check-expressions.test @@ -1278,11 +1278,44 @@ b'%a' % 3 [builtins fixtures/primitives.pyi] [typing fixtures/typing-medium.pyi] +[case testStringInterPolationCPython2] +# flags: --py2 --no-strict-optional +'%c' % 1 +'%c' % 1.0 # E: Incompatible types in string interpolation (expression has type "float", placeholder has type "Union[int, str]") +'%c' % 's' +'%c' % '' # E: "%c" requires int or char +'%c' % 'ab' # E: "%c" requires int or char +'%c' % b'a' +[builtins_py2 fixtures/python2.pyi] + [case testStringInterpolationC] +# flags: --python-version 3.6 '%c' % 1 +'%c' % 1.0 # E: Incompatible types in string interpolation (expression has type "float", placeholder has type "Union[int, str]") '%c' % 's' '%c' % '' # E: "%c" requires int or char '%c' % 'ab' # E: "%c" requires int or char +'%c' % b'a' # E: Incompatible types in string interpolation (expression has type "bytes", placeholder has type "Union[int, str]") +[builtins fixtures/primitives.pyi] + +[case testBytesInterPolationCPython2] +# flags: --py2 --no-strict-optional +b'%c' % 1 +b'%c' % 1.0 # E: Incompatible types in string interpolation (expression has type "float", placeholder has type "Union[int, str]") +b'%c' % 's' +b'%c' % '' # E: "%c" requires int or char +b'%c' % 'ab' # E: "%c" requires int or char +b'%c' % b'a' +[builtins_py2 fixtures/python2.pyi] + +[case testBytesInterpolationC] +# flags: --python-version 3.6 +b'%c' % 1 +b'%c' % 1.0 # E: Incompatible types in string interpolation (expression has type "float", placeholder has type "Union[int, bytes]") +b'%c' % 's' # E: Incompatible types in string interpolation (expression has type "str", placeholder has type "Union[int, bytes]") +b'%c' % '' # E: Incompatible types in string interpolation (expression has type "str", placeholder has type "Union[int, bytes]") +b'%c' % 'ab' # E: Incompatible types in string interpolation (expression has type "str", placeholder has type "Union[int, bytes]") +b'%c' % b'a' [builtins fixtures/primitives.pyi] [case testStringInterpolationMappingTypes] @@ -1540,7 +1573,7 @@ x: Union[Good, Bad] class C: ... -'{:c}'.format(C()) # E: Incompatible types in string interpolation (expression has type "C", placeholder has type "Union[int, float, str]") +'{:c}'.format(C()) # E: Incompatible types in string interpolation (expression has type "C", placeholder has type "Union[int, str]") x: str '{:c}'.format(x) [builtins fixtures/primitives.pyi] diff --git a/test-data/unit/fixtures/python2.pyi b/test-data/unit/fixtures/python2.pyi index 44cb9de9be1da..51af59c8bd455 100644 --- a/test-data/unit/fixtures/python2.pyi +++ b/test-data/unit/fixtures/python2.pyi @@ -18,6 +18,8 @@ class unicode: def format(self, *args, **kwars) -> unicode: ... class bool(int): pass +bytes = str + T = TypeVar('T') S = TypeVar('S') class list(Iterable[T], Generic[T]): From 74b63ffe56a608262a317f365aa873976ec95dad Mon Sep 17 00:00:00 2001 From: Jingchen Ye <97littleleaf11@gmail.com> Date: Sun, 25 Jul 2021 23:36:03 +0800 Subject: [PATCH 2/6] Remove cast --- mypy/checkstrformat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/checkstrformat.py b/mypy/checkstrformat.py index e9d3f96002b24..6c66e48afe391 100644 --- a/mypy/checkstrformat.py +++ b/mypy/checkstrformat.py @@ -893,7 +893,7 @@ def check_expr(expr: Expression) -> None: # Python 3 doesn't support b'%c' % str if not (self.chk.options.python_version >= (3, 0) and isinstance(format_expr, BytesExpr)): - if isinstance(expr, (StrExpr, BytesExpr)) and len(cast(StrExpr, expr).value) != 1: + if isinstance(expr, (StrExpr, BytesExpr)) and len(expr.value) != 1: self.msg.requires_int_or_char(context) check_type(type) From 02f4b9d8a1cc06573fba87d1e2a99faee16f17fa Mon Sep 17 00:00:00 2001 From: Jingchen Ye <97littleleaf11@gmail.com> Date: Mon, 26 Jul 2021 15:39:33 +0800 Subject: [PATCH 3/6] Use special type error message with %c --- mypy/checkstrformat.py | 56 +++++++++++++++++---------- mypy/messages.py | 6 +++ test-data/unit/check-expressions.test | 30 ++++++++------ 3 files changed, 60 insertions(+), 32 deletions(-) diff --git a/mypy/checkstrformat.py b/mypy/checkstrformat.py index 6c66e48afe391..a2bd466032212 100644 --- a/mypy/checkstrformat.py +++ b/mypy/checkstrformat.py @@ -40,7 +40,7 @@ from mypy.parse import parse FormatStringExpr = Union[StrExpr, BytesExpr, UnicodeExpr] -Checkers = Tuple[Callable[[Expression], None], Callable[[Type], None]] +Checkers = Tuple[Callable[[Expression], bool], Callable[[Type], None]] MatchMap = Dict[Tuple[int, int], Match[str]] # span -> match @@ -813,10 +813,10 @@ def checkers_for_star(self, context: Context) -> Checkers: """ expected = self.named_type('builtins.int') - def check_type(type: Type) -> None: + def check_type(type: Type) -> bool: expected = self.named_type('builtins.int') - self.chk.check_subtype(type, expected, context, '* wants int', - code=codes.STRING_FORMATTING) + return self.chk.check_subtype(type, expected, context, '* wants int', + code=codes.STRING_FORMATTING) def check_expr(expr: Expression) -> None: type = self.accept(expr, expected) @@ -824,11 +824,11 @@ def check_expr(expr: Expression) -> None: return check_expr, check_type - def check_placeholder_type(self, typ: Type, expected_type: Type, context: Context) -> None: - self.chk.check_subtype(typ, expected_type, context, - message_registry.INCOMPATIBLE_TYPES_IN_STR_INTERPOLATION, - 'expression has type', 'placeholder has type', - code=codes.STRING_FORMATTING) + def check_placeholder_type(self, typ: Type, expected_type: Type, context: Context) -> bool: + return self.chk.check_subtype(typ, expected_type, context, + message_registry.INCOMPATIBLE_TYPES_IN_STR_INTERPOLATION, + 'expression has type', 'placeholder has type', + code=codes.STRING_FORMATTING) def checkers_for_regular_type(self, type: str, context: Context, @@ -840,11 +840,12 @@ def checkers_for_regular_type(self, type: str, if expected_type is None: return None - def check_type(typ: Type) -> None: + def check_type(typ: Type) -> bool: assert expected_type is not None - self.check_placeholder_type(typ, expected_type, context) - if type == 's': - self.check_s_special_cases(expr, typ, context) + ret = self.check_placeholder_type(typ, expected_type, context) + if ret and type == 's': + ret = self.check_s_special_cases(expr, typ, context) + return ret def check_expr(expr: Expression) -> None: type = self.accept(expr, expected_type) @@ -852,7 +853,7 @@ def check_expr(expr: Expression) -> None: return check_expr, check_type - def check_s_special_cases(self, expr: FormatStringExpr, typ: Type, context: Context) -> None: + def check_s_special_cases(self, expr: FormatStringExpr, typ: Type, context: Context) -> bool: """Additional special cases for %s in bytes vs string context.""" if isinstance(expr, StrExpr): # Couple special cases for string formatting. @@ -862,6 +863,7 @@ def check_s_special_cases(self, expr: FormatStringExpr, typ: Type, context: Cont "On Python 3 '%s' % b'abc' produces \"b'abc'\", not 'abc'; " "use '%r' % b'abc' if this is desired behavior", context, code=codes.STR_BYTES_PY3) + return False if self.chk.options.python_version < (3, 0): if has_type_component(typ, 'builtins.unicode'): self.unicode_upcast = True @@ -871,6 +873,8 @@ def check_s_special_cases(self, expr: FormatStringExpr, typ: Type, context: Cont if has_type_component(typ, 'builtins.str'): self.msg.fail("On Python 3 b'%s' requires bytes, not string", context, code=codes.STRING_FORMATTING) + return False + return True def checkers_for_c_type(self, type: str, context: Context, @@ -882,20 +886,30 @@ def checkers_for_c_type(self, type: str, if expected_type is None: return None - def check_type(type: Type) -> None: + def check_type(type: Type) -> bool: assert expected_type is not None - self.check_placeholder_type(type, expected_type, context) + if self.chk.options.python_version >= (3, 0) and isinstance(format_expr, BytesExpr): + err_msg = '"%c" requires an integer in range(256) or a single byte' + else: + err_msg = '"%c" requires int or char' + return self.chk.check_subtype(type, expected_type, context, err_msg, + 'expression has type', + code=codes.STRING_FORMATTING) def check_expr(expr: Expression) -> None: """int, or str with length 1""" type = self.accept(expr, expected_type) - # TODO: Use the same the error message when incompatible types match %c - # Python 3 doesn't support b'%c' % str - if not (self.chk.options.python_version >= (3, 0) - and isinstance(format_expr, BytesExpr)): + # We need further check with expr to make sure that + # it has exact one char or one single byte. + if check_type(type): + # Python 3 doesn't support b'%c' % str + if (self.chk.options.python_version >= (3, 0) and isinstance(format_expr, BytesExpr) + and isinstance(expr, BytesExpr) and len(expr.value) != 1): + self.msg.requires_int_or_single_byte(context) + return + # In Python 2, b'%c' is the same as '%c' if isinstance(expr, (StrExpr, BytesExpr)) and len(expr.value) != 1: self.msg.requires_int_or_char(context) - check_type(type) return check_expr, check_type diff --git a/mypy/messages.py b/mypy/messages.py index e1f1f90206635..dcece4ae2c3a7 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -961,6 +961,12 @@ def string_interpolation_with_star_and_key(self, context: Context) -> None: self.fail('String interpolation contains both stars and mapping keys', context, code=codes.STRING_FORMATTING) + def requires_int_or_single_byte(self, context: Context, + format_call: bool = False) -> None: + self.fail('"{}c" requires an integer in range(256) or a single byte' \ + .format(':' if format_call else '%'), + context, code=codes.STRING_FORMATTING) + def requires_int_or_char(self, context: Context, format_call: bool = False) -> None: self.fail('"{}c" requires int or char'.format(':' if format_call else '%'), diff --git a/test-data/unit/check-expressions.test b/test-data/unit/check-expressions.test index 84727aea8336e..ff3a5efde6adb 100644 --- a/test-data/unit/check-expressions.test +++ b/test-data/unit/check-expressions.test @@ -1281,41 +1281,49 @@ b'%a' % 3 [case testStringInterPolationCPython2] # flags: --py2 --no-strict-optional '%c' % 1 -'%c' % 1.0 # E: Incompatible types in string interpolation (expression has type "float", placeholder has type "Union[int, str]") +'%c' % 1.0 # E: "%c" requires int or char (expression has type "float") '%c' % 's' -'%c' % '' # E: "%c" requires int or char +'%c' % '' # E: "%c" requires int or char '%c' % 'ab' # E: "%c" requires int or char '%c' % b'a' +'%c' % b'' # E: "%c" requires int or char +'%c' % b'ab' # E: "%c" requires int or char [builtins_py2 fixtures/python2.pyi] [case testStringInterpolationC] # flags: --python-version 3.6 '%c' % 1 -'%c' % 1.0 # E: Incompatible types in string interpolation (expression has type "float", placeholder has type "Union[int, str]") +'%c' % 1.0 # E: "%c" requires int or char (expression has type "float") '%c' % 's' -'%c' % '' # E: "%c" requires int or char +'%c' % '' # E: "%c" requires int or char '%c' % 'ab' # E: "%c" requires int or char -'%c' % b'a' # E: Incompatible types in string interpolation (expression has type "bytes", placeholder has type "Union[int, str]") +'%c' % b'a' # E: "%c" requires int or char (expression has type "bytes") +'%c' % b'' # E: "%c" requires int or char (expression has type "bytes") +'%c' % b'ab' # E: "%c" requires int or char (expression has type "bytes") [builtins fixtures/primitives.pyi] [case testBytesInterPolationCPython2] # flags: --py2 --no-strict-optional b'%c' % 1 -b'%c' % 1.0 # E: Incompatible types in string interpolation (expression has type "float", placeholder has type "Union[int, str]") +b'%c' % 1.0 # E: "%c" requires int or char (expression has type "float") b'%c' % 's' -b'%c' % '' # E: "%c" requires int or char +b'%c' % '' # E: "%c" requires int or char b'%c' % 'ab' # E: "%c" requires int or char b'%c' % b'a' +b'%c' % b'' # E: "%c" requires int or char +b'%c' % b'aa' # E: "%c" requires int or char [builtins_py2 fixtures/python2.pyi] [case testBytesInterpolationC] # flags: --python-version 3.6 b'%c' % 1 -b'%c' % 1.0 # E: Incompatible types in string interpolation (expression has type "float", placeholder has type "Union[int, bytes]") -b'%c' % 's' # E: Incompatible types in string interpolation (expression has type "str", placeholder has type "Union[int, bytes]") -b'%c' % '' # E: Incompatible types in string interpolation (expression has type "str", placeholder has type "Union[int, bytes]") -b'%c' % 'ab' # E: Incompatible types in string interpolation (expression has type "str", placeholder has type "Union[int, bytes]") +b'%c' % 1.0 # E: "%c" requires an integer in range(256) or a single byte (expression has type "float") +b'%c' % 's' # E: "%c" requires an integer in range(256) or a single byte (expression has type "str") +b'%c' % '' # E: "%c" requires an integer in range(256) or a single byte (expression has type "str") +b'%c' % 'ab' # E: "%c" requires an integer in range(256) or a single byte (expression has type "str") b'%c' % b'a' +b'%c' % b'' # E: "%c" requires an integer in range(256) or a single byte +b'%c' % b'aa' # E: "%c" requires an integer in range(256) or a single byte [builtins fixtures/primitives.pyi] [case testStringInterpolationMappingTypes] From eddc1d325a4046d781de151e62dfbfaf560c8f06 Mon Sep 17 00:00:00 2001 From: Jingchen Ye <97littleleaf11@gmail.com> Date: Mon, 26 Jul 2021 16:07:24 +0800 Subject: [PATCH 4/6] Fix type --- mypy/checkstrformat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/checkstrformat.py b/mypy/checkstrformat.py index 25673b5104b23..31883ee5690aa 100644 --- a/mypy/checkstrformat.py +++ b/mypy/checkstrformat.py @@ -40,7 +40,7 @@ from mypy.parse import parse FormatStringExpr = Union[StrExpr, BytesExpr, UnicodeExpr] -Checkers = Tuple[Callable[[Expression], bool], Callable[[Type], None]] +Checkers = Tuple[Callable[[Expression], None], Callable[[Type], bool]] MatchMap = Dict[Tuple[int, int], Match[str]] # span -> match From 2d248f08dfe934d1e9766be0db0fe56c6a012a21 Mon Sep 17 00:00:00 2001 From: Jingchen Ye <97littleleaf11@gmail.com> Date: Mon, 26 Jul 2021 16:42:43 +0800 Subject: [PATCH 5/6] Fix code style --- mypy/checkstrformat.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy/checkstrformat.py b/mypy/checkstrformat.py index 31883ee5690aa..adf1ba8c071fa 100644 --- a/mypy/checkstrformat.py +++ b/mypy/checkstrformat.py @@ -888,12 +888,12 @@ def check_expr(expr: Expression) -> None: # it has exact one char or one single byte. if check_type(type): # Python 3 doesn't support b'%c' % str - if (self.chk.options.python_version >= (3, 0) and isinstance(format_expr, BytesExpr) + if (self.chk.options.python_version >= (3, 0) + and isinstance(format_expr, BytesExpr) and isinstance(expr, BytesExpr) and len(expr.value) != 1): self.msg.requires_int_or_single_byte(context) - return # In Python 2, b'%c' is the same as '%c' - if isinstance(expr, (StrExpr, BytesExpr)) and len(expr.value) != 1: + elif isinstance(expr, (StrExpr, BytesExpr)) and len(expr.value) != 1: self.msg.requires_int_or_char(context) return check_expr, check_type From edc8f85442ae75f6f37d5c3d83e59e98c352e6f0 Mon Sep 17 00:00:00 2001 From: Jingchen Ye <97littleleaf11@gmail.com> Date: Mon, 26 Jul 2021 18:25:14 +0800 Subject: [PATCH 6/6] Fix code style --- mypy/messages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/messages.py b/mypy/messages.py index dcece4ae2c3a7..57198d0386d65 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -963,7 +963,7 @@ def string_interpolation_with_star_and_key(self, context: Context) -> None: def requires_int_or_single_byte(self, context: Context, format_call: bool = False) -> None: - self.fail('"{}c" requires an integer in range(256) or a single byte' \ + self.fail('"{}c" requires an integer in range(256) or a single byte' .format(':' if format_call else '%'), context, code=codes.STRING_FORMATTING)