add integration test (load all files in typeshed), fix some stuff it found

JelleZijlstra · JelleZijlstra · commit afc14eb24c2f · 2017-04-30T21:44:29.000-07:00
diff --git a/tests/test.py b/tests/test.py
@@ -1,3 +1,4 @@
+import os
 from pathlib import Path
 from typed_ast import ast3
 import typeshed_client
@@ -97,7 +98,8 @@ def test_conditions(self) -> None:
         self.check_conditions({'penguins', 'asyncio', 'new_stuff'}, version=(3, 4))
         self.check_conditions({'penguins', 'yield_from', 'new_stuff'}, version=(3, 3))
         self.check_conditions({'penguins', 'ages_long_past', 'new_stuff'}, version=(3, 2))
-        self.check_conditions({'penguins', 'ages_long_past', 'old_stuff'}, version=(2, 7))
+        self.check_conditions({'penguins', 'ages_long_past', 'old_stuff', 'more_old_stuff'},
+                              version=(2, 7))
 
     def check_conditions(self, names: Set[str], *, version: Tuple[int, int] = (3, 6),
                          platform: str = 'linux') -> None:
@@ -132,5 +134,25 @@ def test_simple(self) -> None:
         self.assertIsInstance(res.get_name(path, 'var'), typeshed_client.NameInfo)
 
 
+class IntegrationTest(unittest.TestCase):
+    """Tests that all files in typeshed are parsed without error."""
+    fake_env = typeshed_client.parser.Env((3, 6), 'linux')
+    fake_path = typeshed_client.ModulePath(('some', 'module'))
+
+    def test(self):
+        typeshed_root = typeshed_client.finder.find_typeshed()
+        for dirpath, _, filenames in os.walk(typeshed_root):
+            for filename in filenames:
+                path = Path(dirpath) / filename
+                if path.suffix != '.pyi':
+                    continue
+                with self.subTest(path=path):
+                    self.check_path(path)
+
+    def check_path(self, path: Path) -> None:
+        ast = typeshed_client.finder.parse_stub_file(path)
+        typeshed_client.parser.parse_ast(ast, self.fake_env, self.fake_path)
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/typeshed/stdlib/2and3/conditions.pyi b/tests/typeshed/stdlib/2and3/conditions.pyi
@@ -22,3 +22,6 @@ if sys.version_info < (3, 0):
     old_stuff: int
 else:
     new_stuff: int
+
+if sys.version_info[0] == 2:
+    more_old_stuff: int
diff --git a/typeshed_client/finder.py b/typeshed_client/finder.py
@@ -58,6 +58,10 @@ def get_stub_ast(module_name: str,
     path = get_stub_file(module_name, version=version)
     if path is None:
         return None
+    return parse_stub_file(path)
+
+
+def parse_stub_file(path: Path) -> ast3.AST:
     text = path.read_text()
     # Always parse stubs as Python 3.6
     return ast3.parse(text, filename=str(path), feature_version=6)
diff --git a/typeshed_client/parser.py b/typeshed_client/parser.py
@@ -1,12 +1,15 @@
 """This module is responsible for parsing a stub AST into a dictionary of names."""
 
+import logging
 from mypy_extensions import NoReturn
 import sys
 from typed_ast import ast3
 from typing import Any, Dict, Iterable, List, NamedTuple, NewType, Optional, Tuple, Union
 
 from . import finder
 
+log = logging.getLogger(__name__)
+
 
 class InvalidStub(Exception):
     pass
@@ -54,10 +57,11 @@ def parse_ast(ast: ast3.AST, env: Env, module_name: ModulePath) -> NameDict:
     for info in names:
         if info.name in name_dict:
             if isinstance(info.ast, ImportedName) or info.child_nodes:
-                raise InvalidStub('Cannot overload a class or an imported name')
+                log.warning('Cannot overload a class or an imported name: %s', info)
+                continue
             existing = name_dict[info.name]
             if isinstance(existing.ast, ImportedName) or existing.child_nodes:
-                raise InvalidStub('Cannot overload a class or an imported name')
+                log.warning('Cannot overload a class or an imported name: %s', existing)
             elif isinstance(existing.ast, OverloadedName):
                 existing.ast.definitions.append(info.ast)
             else:
@@ -172,6 +176,9 @@ def visit_Num(self, node: ast3.Num) -> int:
     def visit_Str(self, node: ast3.Str) -> str:
         return node.s
 
+    def visit_Index(self, node: ast3.Index) -> int:
+        return self.visit(node.value)
+
     def visit_Tuple(self, node: ast3.Tuple) -> Tuple[Any, ...]:
         return tuple(self.visit(elt) for elt in node.elts)
 
