Make it possible to package and install mypyc (mypyc/mypyc#521)

msullivan · web-flow · commit a1dbad16ff42 · 2019-01-17T19:07:20.000-06:00
There are a couple pieces to this:
 * Write a setup.py - this is heavily cribbed from mypy's
 * Move lib-rt and external/mypy underneath the mypyc directory so
   that they can be mypyc's "package_data"
 * Have `mypyc/__init__.py` adjust sys.path so that the copy of mypy
   is found
diff --git a/.gitignore b/.gitignore
@@ -13,3 +13,4 @@ test_capi
 /dmypy.json
 /.dmypy.json
 /.mypyc-flake8-cache.json
+/build
diff --git a/.gitmodules b/.gitmodules
@@ -1,3 +1,3 @@
-[submodule "mypy"]
-	path = external/mypy
+[submodule "mypyc/external/mypy"]
+	path = mypyc/external/mypy
 	url = https://github.com/python/mypy
diff --git a/.travis.yml b/.travis.yml
@@ -64,11 +64,10 @@ before_install: |
   pip install -U pip setuptools wheel
 
 install:
-  - pip install -r external/mypy/test-requirements.txt
+  - pip install -r mypyc/external/mypy/test-requirements.txt
 
 script: |
   set -e
-  export PYTHONPATH=`pwd`/external/mypy
   if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
     export PYTHONCONFIG="$PYTHONDIR/bin/python-config"
     export LD_LIBRARY_PATH="$PYTHONDIR/lib"
diff --git a/README.md b/README.md
@@ -46,17 +46,13 @@ Optionally create a virtualenv (recommended):
 
 Then install the dependencies:
 
-    $ python3 -m pip install -r external/mypy/test-requirements.txt
-
-You need to have the `mypy` subdirectory in your `PYTHONPATH`:
-
-    $ export PYTHONPATH=`pwd`/external/mypy
+    $ python3 -m pip install -r mypyc/external/mypy/test-requirements.txt
 
 Now you can run the tests:
 
     $ pytest mypyc
 
-Look at the [issue tracker](https://github.com/JukkaL/mypyc/issues)
+Look at the [issue tracker](https://github.com/mypyc/mypyc/issues)
 for things to work on. Please express your interest in working on an
 issue by adding a comment before doing any significant work, since
 development is currently very active and there is real risk of duplicate
diff --git a/appveyor.yml b/appveyor.yml
@@ -16,12 +16,11 @@ environment:
 install:
     - "git submodule update --init --recursive"
     - "%PYTHON%\\python.exe -m pip install -U pip setuptools wheel"
-    - "%PYTHON%\\python.exe -m pip install -U -r external/mypy/test-requirements.txt"
+    - "%PYTHON%\\python.exe -m pip install -U -r mypyc/external/mypy/test-requirements.txt"
 
 build: off
 
 test_script:
-    - "SET PYTHONPATH=%CD%\\external\\mypy"
     - "%PYTHON%\\python.exe -m pytest mypyc"
 
 skip_commits:
diff --git a/conftest.py b/conftest.py
@@ -1,3 +1,6 @@
+# need to import mypyc to trigger the path shenanigans that cause mypy to be found
+import mypyc
+
 pytest_plugins = [
     'mypy.test.data',
 ]
diff --git a/mypyc/__init__.py b/mypyc/__init__.py
@@ -0,0 +1,17 @@
+import sys
+import os.path
+
+# mypyc depends on a copy of mypy installed as package-data that won't
+# live in the normal python path (to avoid potential conflicts with an
+# installed mypy). In order to make sure we find it, we perform
+# sys.path shenanigans here when mypyc is first imported.
+base_path = os.path.dirname(__file__)
+mypy_path = os.path.abspath(os.path.join(base_path, 'external', 'mypy'))
+sys.path.insert(0, mypy_path)
+
+# Make sure that we can find mypy and that it is the *right* mypy.  If
+# mypy was imported before mypyc was, our path manipulations might
+# have been too late, and this assert will catch the problem.
+import mypy
+assert mypy.__file__ == os.path.join(mypy_path, 'mypy', '__init__.py'), (
+    "Found a mypy other than the one packaged with mypyc!")
diff --git a/mypyc/build.py b/mypyc/build.py
@@ -33,9 +33,6 @@
 if MYPY:
     from typing import NoReturn
 
-base_path = os.path.join(os.path.dirname(__file__), '..')
-sys.path.insert(0, os.path.join(base_path, 'external/mypy'))
-
 from mypy.main import process_options
 from mypy.errors import CompileError
 from mypy.options import Options
@@ -224,7 +221,7 @@ def shared_lib_name(modules: List[str]) -> str:
 
 def include_dir() -> str:
     """Find the path of the lib-rt dir that needs to be included"""
-    return os.path.join(os.path.abspath(os.path.dirname(__file__)), '..', 'lib-rt')
+    return os.path.join(os.path.abspath(os.path.dirname(__file__)), 'lib-rt')
 
 
 def generate_c(sources: List[BuildSource], options: Options,
diff --git a/mypyc/external/mypy b/mypyc/external/mypy
diff --git a/mypyc/lib-rt/CPy.c b/mypyc/lib-rt/CPy.c
diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h
diff --git a/mypyc/lib-rt/Makefile b/mypyc/lib-rt/Makefile
@@ -7,18 +7,18 @@ all: test_capi
 OS := $(shell uname)
 PYTHONCONFIG ?= python3-config
 
-CFLAGS += -I ../external/googletest -I ../external/googletest/include -std=c++11
+CFLAGS += -I ../../external/googletest -I ../../external/googletest/include -std=c++11
 
 CFLAGS += $(shell $(PYTHONCONFIG) --cflags)
 CFLAGS += -fPIC
 
 LDFLAGS += $(shell $(PYTHONCONFIG) --ldflags)
 
-test_capi.o: test_capi.cc CPy.h pythonsupport.h mypyc_util.h Makefile ../mypyc/test/test_external.py
+test_capi.o: test_capi.cc CPy.h pythonsupport.h mypyc_util.h Makefile ../test/test_external.py
 	c++ -c -o test_capi.o $(CFLAGS) test_capi.cc
 
 test_capi: test_capi.o
-	c++ -o test_capi test_capi.o ../external/googletest/make/gtest-all.o $(LDFLAGS)
+	c++ -o test_capi test_capi.o ../../external/googletest/make/gtest-all.o $(LDFLAGS)
 
 test: test_capi
 	./test_capi --gtest_print_time=0
diff --git a/mypyc/lib-rt/mypyc_util.h b/mypyc/lib-rt/mypyc_util.h
diff --git a/mypyc/lib-rt/pythonsupport.h b/mypyc/lib-rt/pythonsupport.h
diff --git a/mypyc/lib-rt/test_capi.cc b/mypyc/lib-rt/test_capi.cc
diff --git a/mypyc/test/test_external.py b/mypyc/test/test_external.py
@@ -30,13 +30,13 @@ def test_c_unit_test(self) -> None:
             env['GTEST_COLOR'] = 'yes'  # Use fancy colors
         status = subprocess.call(['make', 'test'],
                                  env=env,
-                                 cwd=os.path.join(base_dir, 'lib-rt'))
+                                 cwd=os.path.join(base_dir, 'mypyc', 'lib-rt'))
         if status != 0:
             raise AssertionError("make test: C unit test failure")
 
     def test_self_type_check(self) -> None:
         """Use the bundled mypy (in git submodule) to type check mypyc."""
-        mypy_dir = os.path.join(base_dir, 'external', 'mypy')
+        mypy_dir = os.path.join(base_dir, 'mypyc', 'external', 'mypy')
         if not os.path.exists(os.path.join(mypy_dir, 'mypy', 'typeshed', 'stdlib')):
             raise AssertionError('Submodule mypy/mypy/typeshed not ready')
         env = os.environ.copy()
@@ -80,6 +80,7 @@ def cached_flake8(dir: str) -> int:
     else:
         cache = {}
     files = glob.glob('%s/**/*.py' % dir, recursive=True)
+    files = [f for f in files if not f.startswith(os.path.join(dir, 'external', 'mypy'))]
     hashes = {}
     for fn in files:
         with open(fn, 'rb') as fb:
diff --git a/mypyc/version.py b/mypyc/version.py
@@ -0,0 +1,13 @@
+import os.path
+
+from mypy import git
+
+__version__ = '0.0.1+dev'
+base_version = __version__
+
+mypy_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+if __version__.endswith('+dev') and git.is_git_repo(mypy_dir) and git.have_git():
+    __version__ += '.' + git.git_revision(mypy_dir).decode('utf-8')
+    if git.is_dirty(mypy_dir):
+        __version__ += '.dirty'
+del mypy_dir
diff --git a/pytest.ini b/pytest.ini
@@ -1,2 +1,3 @@
 [pytest]
 testpaths = 'mypyc/test'
+norecursedirs = mypyc/external
diff --git a/setup.py b/setup.py
@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+
+import glob
+import os
+import os.path
+import sys
+
+if sys.version_info < (3, 5, 0):
+    sys.stderr.write("ERROR: You need Python 3.5 or later to use mypyc.\n")
+    exit(1)
+
+# This requires setuptools when building; setuptools is not needed
+# when installing from a wheel file.
+from setuptools import setup
+from setuptools.command.build_py import build_py
+from mypyc.version import __version__ as version
+
+description = 'Compiler from type annotated Python to C extensions'
+long_description = '''
+mypyc -- Compiler from type annotated Python to C extensions
+=========================================
+
+'''.lstrip()
+
+
+def find_package_data(base, globs):
+    """Find all interesting data files, for setup(package_data=)
+
+    Arguments:
+      root:  The directory to search in.
+      globs: A list of glob patterns to accept files.
+    """
+
+    rv_dirs = [root for root, dirs, files in os.walk(base)]
+    rv = []
+    for rv_dir in rv_dirs:
+        files = []
+        for pat in globs:
+            files += glob.glob(os.path.join(rv_dir, pat))
+        if not files:
+            continue
+        rv.extend([os.path.relpath(f, 'mypyc') for f in files])
+    return rv
+
+
+class CustomPythonBuild(build_py):
+    def pin_version(self):
+        path = os.path.join(self.build_lib, 'mypy')
+        self.mkpath(path)
+        with open(os.path.join(path, 'version.py'), 'w') as stream:
+            stream.write('__version__ = "{}"\n'.format(version))
+
+    def run(self):
+        self.execute(self.pin_version, ())
+        build_py.run(self)
+
+
+cmdclass = {'build_py': CustomPythonBuild}
+
+package_data = []
+
+package_data += find_package_data(
+    os.path.join('mypyc', 'external', 'mypy', 'mypy'), ['*.py', '*.pyi'])
+package_data += find_package_data(
+    os.path.join('mypyc', 'lib-rt'), ['*.c', '*.h'])
+
+classifiers = [
+    'Development Status :: 3 - Alpha',
+    'Environment :: Console',
+    'Intended Audience :: Developers',
+    'License :: OSI Approved :: MIT License',
+    'Programming Language :: Python :: 3',
+    'Programming Language :: Python :: 3.5',
+    'Programming Language :: Python :: 3.6',
+    'Programming Language :: Python :: 3.7',
+    'Topic :: Software Development',
+]
+
+setup(name='mypyc',
+      version=version,
+      description=description,
+      long_description=long_description,
+      author='Jukka Lehtosalo',
+      author_email='jukka.lehtosalo@iki.fi',
+      url='https://github.com/mypyc/mypyc',
+      license='MIT License',
+      py_modules=[],
+      packages=['mypyc', 'mypyc.test'],
+      package_data={'mypyc': package_data},
+      scripts=['scripts/mypyc'],
+      classifiers=classifiers,
+      cmdclass=cmdclass,
+      install_requires = ['typed-ast >= 1.2.0, < 1.3.0',
+                          'mypy_extensions >= 0.4.0, < 0.5.0',
+                          ],
+      include_package_data=True,
+      )

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,6 @@`
	`1`	`+# need to import mypyc to trigger the path shenanigans that cause mypy to be found`
	`2`	`+import mypyc`
	`3`	`+`
`1`	`4`	`pytest_plugins = [`
`2`	`5`	`'mypy.test.data',`
`3`	`6`	`]`