From 0a47ddbd5130517ec726b6c4812d8f1bb16f0a95 Mon Sep 17 00:00:00 2001 From: Chad Dombrova Date: Mon, 10 Oct 2016 22:09:40 -0700 Subject: [PATCH 1/8] Rough draft of pluggable system for producing types from docstrings --- mypy/docstrings.py | 27 +++++++++++++++++++++++++++ mypy/fastparse.py | 12 +++++++++++- mypy/fastparse2.py | 12 +++++++++++- mypy/parse.py | 13 +++++++++++-- 4 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 mypy/docstrings.py diff --git a/mypy/docstrings.py b/mypy/docstrings.py new file mode 100644 index 000000000000..a5faa2496c6f --- /dev/null +++ b/mypy/docstrings.py @@ -0,0 +1,27 @@ +from typing import List, Tuple, Dict +from mypy.types import Type, CallableType, AnyType +from mypy.nodes import Argument + + +def parse_docstring(docstring: str, line: int) -> Tuple[Dict[str, Type], Type]: + """ + Parse a docstring and return type representations. This function can + be overridden by third-party tools which aim to add typing via docstrings. + + Returns a 2-tuple: dictionary of arg name to Type, and return Type. + """ + return None, None + + +def make_callable(args: List[Argument], type_map: Dict[str, Type], + ret_type: Type) -> CallableType: + if type_map is not None: + arg_kinds = [arg.kind for arg in args] + arg_names = [arg.variable.name() for arg in args] + arg_types = [type_map.get(name) for name in arg_names] + + return CallableType([a if a is not None else AnyType() for a in arg_types], + arg_kinds, + arg_names, + ret_type, None, + is_ellipsis_args=False) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index f68d1b0279d0..8042a2e0f433 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -1,4 +1,5 @@ from functools import wraps +from inspect import cleandoc import sys from typing import Tuple, Union, TypeVar, Callable, Sequence, Optional, Any, cast, List @@ -22,6 +23,7 @@ ) from mypy import defaults from mypy import experiments +from mypy import docstrings from mypy.errors import Errors try: @@ -287,7 +289,15 @@ def do_func_def(self, n: Union[ast35.FunctionDef, ast35.AsyncFunctionDef], else: arg_types = [a.type_annotation for a in args] return_type = TypeConverter(line=n.lineno).visit(n.returns) - + # docstrings + if not any(arg_types) and return_type is None: + doc = ast35.get_docstring(n, clean=False) + if doc: + doc = cleandoc(doc.decode('unicode_escape')) + type_map, rtype = docstrings.parse_docstring(doc, n.lineno) + if type_map is not None: + arg_types = [type_map.get(name) for name in arg_names] + return_type = rtype for arg, arg_type in zip(args, arg_types): self.set_type_optional(arg_type, arg.initializer) diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py index 76e95bb4cd55..abf9b682a46f 100644 --- a/mypy/fastparse2.py +++ b/mypy/fastparse2.py @@ -15,6 +15,7 @@ two in a typesafe way. """ from functools import wraps +from inspect import cleandoc import sys from typing import Tuple, Union, TypeVar, Callable, Sequence, Optional, Any, cast, List @@ -39,6 +40,7 @@ from mypy import experiments from mypy.errors import Errors from mypy.fastparse import TypeConverter, TypeCommentParseError +from mypy import docstrings try: from typed_ast import ast27 @@ -293,7 +295,15 @@ def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement: else: arg_types = [a.type_annotation for a in args] return_type = converter.visit(None) - + # docstrings + if not any(arg_types) and return_type is None: + doc = ast27.get_docstring(n, clean=False) + if doc: + doc = cleandoc(doc.decode('unicode_escape')) + type_map, rtype = docstrings.parse_docstring(doc, n.lineno) + if type_map is not None: + arg_types = [type_map.get(name) for name in arg_names] + return_type = rtype for arg, arg_type in zip(args, arg_types): self.set_type_optional(arg_type, arg.initializer) diff --git a/mypy/parse.py b/mypy/parse.py index 02aa292ecbbf..67c7636a498d 100644 --- a/mypy/parse.py +++ b/mypy/parse.py @@ -5,6 +5,7 @@ """ import re +from inspect import cleandoc from typing import List, Tuple, Set, cast, Union, Optional @@ -38,6 +39,7 @@ from mypy.options import Options from mypy import experiments +from mypy import docstrings class ParseError(Exception): pass @@ -417,7 +419,7 @@ def parse_function(self, no_type_checks: bool = False) -> FuncDef: arg_kinds = [arg.kind for arg in args] arg_names = [arg.variable.name() for arg in args] - body, comment_type = self.parse_block(allow_type=True) + body, comment_type = self.parse_block(args=args) # Potentially insert extra assignment statements to the beginning of the # body, used to decompose Python 2 tuple arguments. body.body[:0] = extra_stmts @@ -831,7 +833,7 @@ def construct_function_type(self, args: List[Argument], ret_type: Type, # Parsing statements - def parse_block(self, allow_type: bool = False) -> Tuple[Block, Type]: + def parse_block(self, args: List[Argument]=None) -> Tuple[Block, Type]: colon = self.expect(':') if not isinstance(self.current(), Break): # Block immediately after ':'. @@ -854,6 +856,13 @@ def parse_block(self, allow_type: bool = False) -> Tuple[Block, Type]: brk = self.expect_break() type = self.parse_type_comment(brk, signature=True) self.expect_indent() + if args is not None: + cur = self.current() + if type is None and isinstance(cur, StrLit): + type_map, ret_type = docstrings.parse_docstring( + cleandoc(cur.parsed()), cur.line) + type = docstrings.make_callable(args, type_map, ret_type) + stmt_list = [] # type: List[Statement] while (not isinstance(self.current(), Dedent) and not isinstance(self.current(), Eof)): From 89e2a1215153e3bdb56e71fcdde1810256db3c23 Mon Sep 17 00:00:00 2001 From: Chad Dombrova Date: Fri, 14 Oct 2016 01:30:52 -0400 Subject: [PATCH 2/8] Revert changes to parse.py since it is deprecated. --- mypy/parse.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/mypy/parse.py b/mypy/parse.py index 67c7636a498d..02aa292ecbbf 100644 --- a/mypy/parse.py +++ b/mypy/parse.py @@ -5,7 +5,6 @@ """ import re -from inspect import cleandoc from typing import List, Tuple, Set, cast, Union, Optional @@ -39,7 +38,6 @@ from mypy.options import Options from mypy import experiments -from mypy import docstrings class ParseError(Exception): pass @@ -419,7 +417,7 @@ def parse_function(self, no_type_checks: bool = False) -> FuncDef: arg_kinds = [arg.kind for arg in args] arg_names = [arg.variable.name() for arg in args] - body, comment_type = self.parse_block(args=args) + body, comment_type = self.parse_block(allow_type=True) # Potentially insert extra assignment statements to the beginning of the # body, used to decompose Python 2 tuple arguments. body.body[:0] = extra_stmts @@ -833,7 +831,7 @@ def construct_function_type(self, args: List[Argument], ret_type: Type, # Parsing statements - def parse_block(self, args: List[Argument]=None) -> Tuple[Block, Type]: + def parse_block(self, allow_type: bool = False) -> Tuple[Block, Type]: colon = self.expect(':') if not isinstance(self.current(), Break): # Block immediately after ':'. @@ -856,13 +854,6 @@ def parse_block(self, args: List[Argument]=None) -> Tuple[Block, Type]: brk = self.expect_break() type = self.parse_type_comment(brk, signature=True) self.expect_indent() - if args is not None: - cur = self.current() - if type is None and isinstance(cur, StrLit): - type_map, ret_type = docstrings.parse_docstring( - cleandoc(cur.parsed()), cur.line) - type = docstrings.make_callable(args, type_map, ret_type) - stmt_list = [] # type: List[Statement] while (not isinstance(self.current(), Dedent) and not isinstance(self.current(), Eof)): From d44291cf21bbeee453ddd15924622f65d0e18a2a Mon Sep 17 00:00:00 2001 From: Chad Dombrova Date: Fri, 14 Oct 2016 22:06:33 -0400 Subject: [PATCH 3/8] Add basic system for registering hooks and use it for docstring parser. --- mypy/docstrings.py | 27 ------------------------- mypy/fastparse.py | 49 +++++++++++++++++++++++++++++++++++++--------- mypy/fastparse2.py | 20 ++++++++++--------- mypy/hooks.py | 28 ++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 45 deletions(-) delete mode 100644 mypy/docstrings.py create mode 100644 mypy/hooks.py diff --git a/mypy/docstrings.py b/mypy/docstrings.py deleted file mode 100644 index a5faa2496c6f..000000000000 --- a/mypy/docstrings.py +++ /dev/null @@ -1,27 +0,0 @@ -from typing import List, Tuple, Dict -from mypy.types import Type, CallableType, AnyType -from mypy.nodes import Argument - - -def parse_docstring(docstring: str, line: int) -> Tuple[Dict[str, Type], Type]: - """ - Parse a docstring and return type representations. This function can - be overridden by third-party tools which aim to add typing via docstrings. - - Returns a 2-tuple: dictionary of arg name to Type, and return Type. - """ - return None, None - - -def make_callable(args: List[Argument], type_map: Dict[str, Type], - ret_type: Type) -> CallableType: - if type_map is not None: - arg_kinds = [arg.kind for arg in args] - arg_names = [arg.variable.name() for arg in args] - arg_types = [type_map.get(name) for name in arg_names] - - return CallableType([a if a is not None else AnyType() for a in arg_types], - arg_kinds, - arg_names, - ret_type, None, - is_ellipsis_args=False) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 8042a2e0f433..89b5e2c38158 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -1,6 +1,6 @@ from functools import wraps -from inspect import cleandoc import sys +import inspect from typing import Tuple, Union, TypeVar, Callable, Sequence, Optional, Any, cast, List from mypy.nodes import ( @@ -23,7 +23,8 @@ ) from mypy import defaults from mypy import experiments -from mypy import docstrings +from mypy import hooks +from mypy.parsetype import parse_str_as_type from mypy.errors import Errors try: @@ -89,6 +90,35 @@ def parse_type_comment(type_comment: str, line: int) -> Type: return TypeConverter(line=line).visit(typ.body) +def parse_docstring(docstring: str, arg_names: List[str], + line: int) -> Optional[Tuple[List[Type], Type]]: + """Parse a docstring and return type representations. + + Returns a 2-tuple: (list of arguments Types, and return Type). + """ + def pop_and_convert(name): + t = type_map.pop(name, None) + if t is None: + return AnyType() + elif isinstance(t, Type): + return t + else: + return parse_str_as_type(t, line) + + docstring_parser = hooks.get_docstring_parser() + if docstring_parser is not None: + type_map = docstring_parser(inspect.cleandoc(docstring), line) + if type_map: + arg_types = [pop_and_convert(name) for name in arg_names] + return_type = pop_and_convert('return') + if type_map: + raise TypeCommentParseError( + 'Arguments parsed from docstring are not present in ' + 'function signature: {}'.format(', '.join(type_map)), + line, 0) + return arg_types, return_type + + def with_line(f: Callable[['ASTConverter', T], U]) -> Callable[['ASTConverter', T], U]: @wraps(f) def wrapper(self: 'ASTConverter', ast: T) -> U: @@ -289,15 +319,16 @@ def do_func_def(self, n: Union[ast35.FunctionDef, ast35.AsyncFunctionDef], else: arg_types = [a.type_annotation for a in args] return_type = TypeConverter(line=n.lineno).visit(n.returns) - # docstrings - if not any(arg_types) and return_type is None: + # hooks + if (not any(arg_types) and return_type is None and + hooks.get_docstring_parser()): doc = ast35.get_docstring(n, clean=False) if doc: - doc = cleandoc(doc.decode('unicode_escape')) - type_map, rtype = docstrings.parse_docstring(doc, n.lineno) - if type_map is not None: - arg_types = [type_map.get(name) for name in arg_names] - return_type = rtype + doc = doc.decode('unicode_escape') + types = parse_docstring(doc, arg_names, n.lineno) + if types is not None: + arg_types, return_type = types + for arg, arg_type in zip(args, arg_types): self.set_type_optional(arg_type, arg.initializer) diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py index abf9b682a46f..061f50a57fe9 100644 --- a/mypy/fastparse2.py +++ b/mypy/fastparse2.py @@ -39,8 +39,9 @@ from mypy import defaults from mypy import experiments from mypy.errors import Errors -from mypy.fastparse import TypeConverter, TypeCommentParseError -from mypy import docstrings +from mypy.fastparse import (TypeConverter, TypeCommentParseError, + parse_docstring) +from mypy import hooks try: from typed_ast import ast27 @@ -295,15 +296,16 @@ def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement: else: arg_types = [a.type_annotation for a in args] return_type = converter.visit(None) - # docstrings - if not any(arg_types) and return_type is None: + # hooks + if (not any(arg_types) and return_type is None and + hooks.get_docstring_parser()): doc = ast27.get_docstring(n, clean=False) if doc: - doc = cleandoc(doc.decode('unicode_escape')) - type_map, rtype = docstrings.parse_docstring(doc, n.lineno) - if type_map is not None: - arg_types = [type_map.get(name) for name in arg_names] - return_type = rtype + doc = doc.decode('unicode_escape') + types = parse_docstring(doc, arg_names, n.lineno) + if types is not None: + arg_types, return_type = types + for arg, arg_type in zip(args, arg_types): self.set_type_optional(arg_type, arg.initializer) diff --git a/mypy/hooks.py b/mypy/hooks.py new file mode 100644 index 000000000000..6431c26447f9 --- /dev/null +++ b/mypy/hooks.py @@ -0,0 +1,28 @@ +from typing import Dict, Optional, Callable, Union +from mypy.types import Type + +hooks = {} # type: Dict[str, Callable] + +docstring_parser_type = Callable[[str, int], Optional[Dict[str, Union[str, Type]]]] + + +def set_docstring_parser(func: docstring_parser_type) -> None: + """Enable the docstring parsing hook. + + The callable must take a docstring for a function along with its line number + (typically passed to mypy.parsetype.parse_str_as_type), and should return + a mapping of argument name to type. The function's return type, if + specified, is stored in the mapping with the special key 'return'. + + The keys of the mapping must be a subset of the arguments of the function + to which the docstring belongs (other than the special 'return' + key); an error will be raised if the mapping contains stray arguments. + + The values of the mapping must be either mypy.types.Type or a valid + PEP484-compatible string which can be converted to a Type. + """ + hooks['docstring_parser'] = func + + +def get_docstring_parser() -> Optional[docstring_parser_type]: + return hooks.get('docstring_parser') From 05c66cdfb998568acd86846d2791182858396713 Mon Sep 17 00:00:00 2001 From: Chad Dombrova Date: Tue, 18 Oct 2016 21:33:24 -0700 Subject: [PATCH 4/8] Address docparser review notes. - simplify and improve registration of mypy.hooks - do not call cleandoc (that's up to the hook user) - doc parser hooks must return strings and not Type --- mypy/fastparse.py | 11 +++-------- mypy/fastparse2.py | 4 ++-- mypy/hooks.py | 41 +++++++++++++---------------------------- 3 files changed, 18 insertions(+), 38 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 89b5e2c38158..18a844a75b9a 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -1,6 +1,5 @@ from functools import wraps import sys -import inspect from typing import Tuple, Union, TypeVar, Callable, Sequence, Optional, Any, cast, List from mypy.nodes import ( @@ -100,14 +99,11 @@ def pop_and_convert(name): t = type_map.pop(name, None) if t is None: return AnyType() - elif isinstance(t, Type): - return t else: return parse_str_as_type(t, line) - docstring_parser = hooks.get_docstring_parser() - if docstring_parser is not None: - type_map = docstring_parser(inspect.cleandoc(docstring), line) + if hooks.docstring_parser is not None: + type_map = hooks.docstring_parser(docstring) if type_map: arg_types = [pop_and_convert(name) for name in arg_names] return_type = pop_and_convert('return') @@ -321,10 +317,9 @@ def do_func_def(self, n: Union[ast35.FunctionDef, ast35.AsyncFunctionDef], return_type = TypeConverter(line=n.lineno).visit(n.returns) # hooks if (not any(arg_types) and return_type is None and - hooks.get_docstring_parser()): + hooks.docstring_parser): doc = ast35.get_docstring(n, clean=False) if doc: - doc = doc.decode('unicode_escape') types = parse_docstring(doc, arg_names, n.lineno) if types is not None: arg_types, return_type = types diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py index 061f50a57fe9..808f14721755 100644 --- a/mypy/fastparse2.py +++ b/mypy/fastparse2.py @@ -298,8 +298,8 @@ def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement: return_type = converter.visit(None) # hooks if (not any(arg_types) and return_type is None and - hooks.get_docstring_parser()): - doc = ast27.get_docstring(n, clean=False) + hooks.docstring_parser): + doc = cast(bytes, ast27.get_docstring(n, clean=False)) if doc: doc = doc.decode('unicode_escape') types = parse_docstring(doc, arg_names, n.lineno) diff --git a/mypy/hooks.py b/mypy/hooks.py index 6431c26447f9..8566c87b7e7d 100644 --- a/mypy/hooks.py +++ b/mypy/hooks.py @@ -1,28 +1,13 @@ -from typing import Dict, Optional, Callable, Union -from mypy.types import Type - -hooks = {} # type: Dict[str, Callable] - -docstring_parser_type = Callable[[str, int], Optional[Dict[str, Union[str, Type]]]] - - -def set_docstring_parser(func: docstring_parser_type) -> None: - """Enable the docstring parsing hook. - - The callable must take a docstring for a function along with its line number - (typically passed to mypy.parsetype.parse_str_as_type), and should return - a mapping of argument name to type. The function's return type, if - specified, is stored in the mapping with the special key 'return'. - - The keys of the mapping must be a subset of the arguments of the function - to which the docstring belongs (other than the special 'return' - key); an error will be raised if the mapping contains stray arguments. - - The values of the mapping must be either mypy.types.Type or a valid - PEP484-compatible string which can be converted to a Type. - """ - hooks['docstring_parser'] = func - - -def get_docstring_parser() -> Optional[docstring_parser_type]: - return hooks.get('docstring_parser') +from typing import Dict, Optional, Callable + +# The docstring_parser hook is called for each function that has a docstring +# and no other type annotations applied, and the callable should accept the +# docstring as an argument and return a mapping of argument name to type. +# +# The function's return type, if specified, is stored in the mapping with the +# special key 'return'. Other than 'return', the keys of the mapping must be +# a subset of the arguments of the function to which the docstring belongs; an +# error will be raised if the mapping contains stray arguments. +# +# The values of the mapping must be valid PEP484-compatible strings. +docstring_parser = None # type: Callable[[str], Optional[Dict[str, str]]] From 6f56a15fe63644e411f9822b88ebd011374b641e Mon Sep 17 00:00:00 2001 From: Chad Dombrova Date: Wed, 19 Oct 2016 15:24:47 -0700 Subject: [PATCH 5/8] Remove unused import --- mypy/fastparse2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py index 808f14721755..a66811e5670e 100644 --- a/mypy/fastparse2.py +++ b/mypy/fastparse2.py @@ -15,7 +15,6 @@ two in a typesafe way. """ from functools import wraps -from inspect import cleandoc import sys from typing import Tuple, Union, TypeVar, Callable, Sequence, Optional, Any, cast, List From 6b0c77b02ee3aa4210dbc864295792b1f9e8685a Mon Sep 17 00:00:00 2001 From: Chad Dombrova Date: Wed, 19 Oct 2016 16:00:29 -0700 Subject: [PATCH 6/8] Remove useless cast. --- mypy/fastparse2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py index a66811e5670e..637cfd2ac524 100644 --- a/mypy/fastparse2.py +++ b/mypy/fastparse2.py @@ -298,7 +298,7 @@ def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement: # hooks if (not any(arg_types) and return_type is None and hooks.docstring_parser): - doc = cast(bytes, ast27.get_docstring(n, clean=False)) + doc = ast27.get_docstring(n, clean=False) if doc: doc = doc.decode('unicode_escape') types = parse_docstring(doc, arg_names, n.lineno) From 775f57526f274bb6032d6276b41d3987287271a3 Mon Sep 17 00:00:00 2001 From: Chad Dombrova Date: Wed, 2 Nov 2016 22:12:34 -0700 Subject: [PATCH 7/8] Use new parser to parse strings into types --- mypy/fastparse.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 18a844a75b9a..5f1d6ab67ca0 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -23,7 +23,6 @@ from mypy import defaults from mypy import experiments from mypy import hooks -from mypy.parsetype import parse_str_as_type from mypy.errors import Errors try: @@ -100,7 +99,7 @@ def pop_and_convert(name): if t is None: return AnyType() else: - return parse_str_as_type(t, line) + return parse_type_comment(t, line=line) if hooks.docstring_parser is not None: type_map = hooks.docstring_parser(docstring) From bbd59642da0855cf0483691295ff46e0fb5e5e09 Mon Sep 17 00:00:00 2001 From: Chad Dombrova Date: Wed, 2 Nov 2016 23:24:17 -0700 Subject: [PATCH 8/8] get tests passing. --- mypy/fastparse.py | 1 + mypy/fastparse2.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 5f1d6ab67ca0..e09e77e59574 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -112,6 +112,7 @@ def pop_and_convert(name): 'function signature: {}'.format(', '.join(type_map)), line, 0) return arg_types, return_type + return None def with_line(f: Callable[['ASTConverter', T], U]) -> Callable[['ASTConverter', T], U]: diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py index 637cfd2ac524..0175d50645fe 100644 --- a/mypy/fastparse2.py +++ b/mypy/fastparse2.py @@ -300,8 +300,8 @@ def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement: hooks.docstring_parser): doc = ast27.get_docstring(n, clean=False) if doc: - doc = doc.decode('unicode_escape') - types = parse_docstring(doc, arg_names, n.lineno) + types = parse_docstring(doc.decode('unicode_escape'), + arg_names, n.lineno) if types is not None: arg_types, return_type = types