From 96041b16ff2d3ca56c0ed5f68d8fb91c1be67104 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 6 Mar 2018 19:25:34 -0800 Subject: [PATCH 1/6] Add docs for python-executable --- docs/source/command_line.rst | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index a72390879911..01b6036f6993 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -360,11 +360,27 @@ Here are some more useful flags: updates the cache, but regular incremental mode ignores cache files written by quick mode. +- ``--python-executable EXECUTABLE`` will have mypy collect type information + from `PEP 561`_ compliant packages installed for the Python executable + ``EXECUTABLE``. If not provided, mypy will use PEP 561 compliant packages + installed for the Python executable running mypy. See + :ref:`installed-packages` for more on making PEP 561 compliant packages. This + flag will attempt to set ``--python-version`` if not already set. + - ``--python-version X.Y`` will make mypy typecheck your code as if it were run under Python version X.Y. Without this option, mypy will default to using whatever version of Python is running mypy. Note that the ``-2`` and ``--py2`` flags are aliases for ``--python-version 2.7``. See - :ref:`version_and_platform_checks` for more about this feature. + :ref:`version_and_platform_checks` for more about this feature. This flag + will attempt to find a Python executable of the corresponding version to + search for `PEP 561`_ compliant packages. If you'd like to disable this, see + ``--no-site-packages`` below. + +- ``--no-site-packages`` will disable searching for `PEP 561`_ compliant + packages. This will also disable searching for a usable Python executable. + Use this flag if mypy cannot find a Python executable for the version of + Python being checked, and you don't need to use PEP 561 typed packages. + Otherwise, use ``--python-executable``. - ``--platform PLATFORM`` will make mypy typecheck your code as if it were run under the the given operating system. Without this option, mypy will @@ -447,6 +463,8 @@ For the remaining flags you can read the full ``mypy -h`` output. Command line flags are liable to change between releases. +.. _PEP 561: https://www.python.org/dev/peps/pep-0561/ + .. _integrating-mypy: Integrating mypy into another Python application From 4483b3eb3119bfa3f86d467c503804fe27dba2b1 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 6 Mar 2018 22:53:15 -0800 Subject: [PATCH 2/6] Add --python-executable and --no-site-packages --- mypy/main.py | 79 ++++++++++++++++++++++++++++++++++++++++++-- mypy/options.py | 1 + mypy/test/helpers.py | 1 + 3 files changed, 79 insertions(+), 2 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index b7c1117ea029..512297b18fbe 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -1,10 +1,12 @@ """Mypy type checker command line tool.""" import argparse +import ast import configparser import fnmatch import os import re +import subprocess import sys import time @@ -205,6 +207,45 @@ def invert_flag_name(flag: str) -> str: return '--no-{}'.format(flag[2:]) +class PythonExecutableInferenceError(Exception): + """Represents a failure to infer the version or executable while searching.""" + + +if sys.platform == 'win32': + def python_executable_prefix(v: str) -> List[str]: + return ['py', '-{}'.format(v)] +else: + def python_executable_prefix(v: str) -> List[str]: + return ['python{}'.format(v)] + + +def _python_version_from_executable(python_executable: str) -> Tuple[int, int]: + try: + check = subprocess.check_output([python_executable, '-c', + 'import sys; print(repr(sys.version_info[:2]))'], + stderr=subprocess.STDOUT).decode() + return ast.literal_eval(check) + except (subprocess.CalledProcessError, FileNotFoundError): + raise PythonExecutableInferenceError( + 'Error: invalid Python executable {}'.format(python_executable)) + + +def _python_executable_from_version(python_version: Tuple[int, int]) -> str: + if sys.version_info[:2] == python_version: + return sys.executable + str_ver = '.'.join(map(str, python_version)) + print(str_ver) + try: + sys_exe = subprocess.check_output(python_executable_prefix(str_ver) + + ['-c', 'import sys; print(sys.executable)'], + stderr=subprocess.STDOUT).decode().strip() + return sys_exe + except (subprocess.CalledProcessError, FileNotFoundError): + raise PythonExecutableInferenceError( + 'Error: failed to find a Python executable matching version {},' + ' perhaps try --python-executable, or --no-site-packages?'.format(python_version)) + + def process_options(args: List[str], require_targets: bool = True, server_options: bool = False, @@ -255,10 +296,16 @@ def add_invertible_flag(flag: str, parser.add_argument('-V', '--version', action='version', version='%(prog)s ' + __version__) parser.add_argument('--python-version', type=parse_version, metavar='x.y', - help='use Python x.y') + help='use Python x.y', dest='special-opts:python_version') + parser.add_argument('--python-executable', action='store', metavar='EXECUTABLE', + help="Python executable whose installed packages will be" + " used in typechecking.", dest='special-opts:python_executable') + parser.add_argument('--no-site-packages', action='store_true', + dest='special-opts:no_site_packages', + help="Do not search for PEP 561 packages in the package directory.") parser.add_argument('--platform', action='store', metavar='PLATFORM', help="typecheck special-cased code for the given OS platform " - "(defaults to sys.platform).") + "(defaults to sys.platform).") parser.add_argument('-2', '--py2', dest='python_version', action='store_const', const=defaults.PYTHON2_VERSION, help="use Python 2 mode") parser.add_argument('--ignore-missing-imports', action='store_true', @@ -482,6 +529,34 @@ def add_invertible_flag(flag: str, print("Warning: --no-fast-parser no longer has any effect. The fast parser " "is now mypy's default and only parser.") + try: + # Infer Python version and/or executable if one is not given + if special_opts.python_executable is not None and special_opts.python_version is not None: + py_exe_ver = _python_version_from_executable(special_opts.python_executable) + if py_exe_ver != special_opts.python_version: + parser.error( + 'Python version {} did not match executable {}, got version {}.'.format( + special_opts.python_version, special_opts.python_executable, py_exe_ver + )) + else: + options.python_version = special_opts.python_version + options.python_executable = special_opts.python_executable + elif special_opts.python_executable is None and special_opts.python_version is not None: + options.python_version = special_opts.python_version + py_exe = None + if not special_opts.no_site_packages: + py_exe = _python_executable_from_version(special_opts.python_version) + options.python_executable = py_exe + elif special_opts.python_version is None and special_opts.python_executable is not None: + options.python_version = _python_version_from_executable( + special_opts.python_executable) + options.python_executable = special_opts.python_executable + except PythonExecutableInferenceError as e: + parser.error(str(e)) + + if special_opts.no_site_packages: + options.python_executable = None + # Check for invalid argument combinations. if require_targets: code_methods = sum(bool(c) for c in [special_opts.modules, diff --git a/mypy/options.py b/mypy/options.py index 5ea251df2c9d..c3d07df08191 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -53,6 +53,7 @@ def __init__(self) -> None: # -- build options -- self.build_type = BuildType.STANDARD self.python_version = sys.version_info[:2] # type: Tuple[int, int] + self.python_executable = sys.executable # type: Optional[str] self.platform = sys.platform self.custom_typing_module = None # type: Optional[str] self.custom_typeshed_dir = None # type: Optional[str] diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py index 267f99e5586b..ad17d8387ace 100644 --- a/mypy/test/helpers.py +++ b/mypy/test/helpers.py @@ -316,6 +316,7 @@ def parse_options(program_text: str, testcase: DataDrivenTestCase, flag_list = None if flags: flag_list = flags.group(1).split() + flag_list.append('--no-site-packages') # the tests shouldn't need an installed Python targets, options = process_options(flag_list, require_targets=False) if targets: # TODO: support specifying targets via the flags pragma From 53ff42c936fa47dd6db6cd200ce4a0eea022df8e Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 6 Mar 2018 23:09:28 -0800 Subject: [PATCH 3/6] Split inference out of process_options --- mypy/main.py | 49 ++++++++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 512297b18fbe..1b5863b226ee 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -246,6 +246,33 @@ def _python_executable_from_version(python_version: Tuple[int, int]) -> str: ' perhaps try --python-executable, or --no-site-packages?'.format(python_version)) +def infer_python_version_and_executable(options: Options, + special_opts: argparse.Namespace + ) -> Options: + # Infer Python version and/or executable if one is not given + if special_opts.python_executable is not None and special_opts.python_version is not None: + py_exe_ver = _python_version_from_executable(special_opts.python_executable) + if py_exe_ver != special_opts.python_version: + raise PythonExecutableInferenceError( + 'Python version {} did not match executable {}, got version {}.'.format( + special_opts.python_version, special_opts.python_executable, py_exe_ver + )) + else: + options.python_version = special_opts.python_version + options.python_executable = special_opts.python_executable + elif special_opts.python_executable is None and special_opts.python_version is not None: + options.python_version = special_opts.python_version + py_exe = None + if not special_opts.no_site_packages: + py_exe = _python_executable_from_version(special_opts.python_version) + options.python_executable = py_exe + elif special_opts.python_version is None and special_opts.python_executable is not None: + options.python_version = _python_version_from_executable( + special_opts.python_executable) + options.python_executable = special_opts.python_executable + return options + + def process_options(args: List[str], require_targets: bool = True, server_options: bool = False, @@ -530,27 +557,7 @@ def add_invertible_flag(flag: str, "is now mypy's default and only parser.") try: - # Infer Python version and/or executable if one is not given - if special_opts.python_executable is not None and special_opts.python_version is not None: - py_exe_ver = _python_version_from_executable(special_opts.python_executable) - if py_exe_ver != special_opts.python_version: - parser.error( - 'Python version {} did not match executable {}, got version {}.'.format( - special_opts.python_version, special_opts.python_executable, py_exe_ver - )) - else: - options.python_version = special_opts.python_version - options.python_executable = special_opts.python_executable - elif special_opts.python_executable is None and special_opts.python_version is not None: - options.python_version = special_opts.python_version - py_exe = None - if not special_opts.no_site_packages: - py_exe = _python_executable_from_version(special_opts.python_version) - options.python_executable = py_exe - elif special_opts.python_version is None and special_opts.python_executable is not None: - options.python_version = _python_version_from_executable( - special_opts.python_executable) - options.python_executable = special_opts.python_executable + options = infer_python_version_and_executable(options, special_opts) except PythonExecutableInferenceError as e: parser.error(str(e)) From f523efa7ab7b8146fad4d55787f066c1be8a8391 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 6 Mar 2018 23:25:26 -0800 Subject: [PATCH 4/6] Change --no-site-packages to --no-infer-executable --- docs/source/command_line.rst | 20 +++++++------------- mypy/main.py | 14 +++++++------- mypy/test/helpers.py | 2 +- 3 files changed, 15 insertions(+), 21 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 01b6036f6993..0dda5af9c637 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -360,27 +360,21 @@ Here are some more useful flags: updates the cache, but regular incremental mode ignores cache files written by quick mode. -- ``--python-executable EXECUTABLE`` will have mypy collect type information - from `PEP 561`_ compliant packages installed for the Python executable - ``EXECUTABLE``. If not provided, mypy will use PEP 561 compliant packages - installed for the Python executable running mypy. See - :ref:`installed-packages` for more on making PEP 561 compliant packages. This - flag will attempt to set ``--python-version`` if not already set. +- ``--python-executable EXECUTABLE`` This flag will attempt to set + ``--python-version`` if not already set based on the interpreter given. - ``--python-version X.Y`` will make mypy typecheck your code as if it were run under Python version X.Y. Without this option, mypy will default to using whatever version of Python is running mypy. Note that the ``-2`` and ``--py2`` flags are aliases for ``--python-version 2.7``. See :ref:`version_and_platform_checks` for more about this feature. This flag - will attempt to find a Python executable of the corresponding version to - search for `PEP 561`_ compliant packages. If you'd like to disable this, see - ``--no-site-packages`` below. + will attempt to find a Python executable of the corresponding version. If + you'd like to disable this, see ``--no-infer-executable`` below. -- ``--no-site-packages`` will disable searching for `PEP 561`_ compliant - packages. This will also disable searching for a usable Python executable. +- ``--no-infer-executable`` will disable searching for a usable Python + executable based on the Python version mypy is using to type check code. Use this flag if mypy cannot find a Python executable for the version of - Python being checked, and you don't need to use PEP 561 typed packages. - Otherwise, use ``--python-executable``. + Python being checked, and don't need mypy to use an executable. - ``--platform PLATFORM`` will make mypy typecheck your code as if it were run under the the given operating system. Without this option, mypy will diff --git a/mypy/main.py b/mypy/main.py index 1b5863b226ee..887dd106e8d5 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -263,7 +263,7 @@ def infer_python_version_and_executable(options: Options, elif special_opts.python_executable is None and special_opts.python_version is not None: options.python_version = special_opts.python_version py_exe = None - if not special_opts.no_site_packages: + if not special_opts.no_executable: py_exe = _python_executable_from_version(special_opts.python_version) options.python_executable = py_exe elif special_opts.python_version is None and special_opts.python_executable is not None: @@ -325,11 +325,11 @@ def add_invertible_flag(flag: str, parser.add_argument('--python-version', type=parse_version, metavar='x.y', help='use Python x.y', dest='special-opts:python_version') parser.add_argument('--python-executable', action='store', metavar='EXECUTABLE', - help="Python executable whose installed packages will be" - " used in typechecking.", dest='special-opts:python_executable') - parser.add_argument('--no-site-packages', action='store_true', - dest='special-opts:no_site_packages', - help="Do not search for PEP 561 packages in the package directory.") + help="Python executable which will be used in typechecking.", + dest='special-opts:python_executable') + parser.add_argument('--no-infer-executable', action='store_true', + dest='special-opts:no_executable', + help="Do not infer a Python executable based on the version.") parser.add_argument('--platform', action='store', metavar='PLATFORM', help="typecheck special-cased code for the given OS platform " "(defaults to sys.platform).") @@ -561,7 +561,7 @@ def add_invertible_flag(flag: str, except PythonExecutableInferenceError as e: parser.error(str(e)) - if special_opts.no_site_packages: + if special_opts.no_executable: options.python_executable = None # Check for invalid argument combinations. diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py index ad17d8387ace..adce623d8af0 100644 --- a/mypy/test/helpers.py +++ b/mypy/test/helpers.py @@ -316,7 +316,7 @@ def parse_options(program_text: str, testcase: DataDrivenTestCase, flag_list = None if flags: flag_list = flags.group(1).split() - flag_list.append('--no-site-packages') # the tests shouldn't need an installed Python + flag_list.append('--no-infer-executable') # the tests shouldn't need an installed Python targets, options = process_options(flag_list, require_targets=False) if targets: # TODO: support specifying targets via the flags pragma From 4d668272d918251c543e96ebca9747fbf43073b4 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 6 Mar 2018 23:43:53 -0800 Subject: [PATCH 5/6] Add --no-infer-executable to subprocessed tests --- mypy/main.py | 2 +- mypy/test/testcmdline.py | 1 + mypy/test/testpythoneval.py | 2 +- runtests.py | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 887dd106e8d5..4f45aba8e3a9 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -243,7 +243,7 @@ def _python_executable_from_version(python_version: Tuple[int, int]) -> str: except (subprocess.CalledProcessError, FileNotFoundError): raise PythonExecutableInferenceError( 'Error: failed to find a Python executable matching version {},' - ' perhaps try --python-executable, or --no-site-packages?'.format(python_version)) + ' perhaps try --python-executable, or --no-infer-executable?'.format(python_version)) def infer_python_version_and_executable(options: Options, diff --git a/mypy/test/testcmdline.py b/mypy/test/testcmdline.py index 57910c1a1dc0..7a3016991cce 100644 --- a/mypy/test/testcmdline.py +++ b/mypy/test/testcmdline.py @@ -47,6 +47,7 @@ def test_python_cmdline(testcase: DataDrivenTestCase) -> None: file.write('{}\n'.format(s)) args = parse_args(testcase.input[0]) args.append('--show-traceback') + args.append('--no-infer-executable') # Type check the program. fixed = [python3_path, os.path.join(testcase.old_cwd, 'scripts', 'mypy')] diff --git a/mypy/test/testpythoneval.py b/mypy/test/testpythoneval.py index 0634442f172a..871753b833de 100644 --- a/mypy/test/testpythoneval.py +++ b/mypy/test/testpythoneval.py @@ -49,7 +49,7 @@ def test_python_evaluation(testcase: DataDrivenTestCase) -> None: version. """ assert testcase.old_cwd is not None, "test was not properly set up" - mypy_cmdline = ['--show-traceback'] + mypy_cmdline = ['--show-traceback', '--no-infer-executable'] py2 = testcase.name.lower().endswith('python2') if py2: mypy_cmdline.append('--py2') diff --git a/runtests.py b/runtests.py index a2a24c29a7ca..0714cf88cabf 100755 --- a/runtests.py +++ b/runtests.py @@ -73,6 +73,7 @@ def add_mypy_cmd(self, name: str, mypy_args: List[str], cwd: Optional[str] = Non return args = [sys.executable, self.mypy] + mypy_args args.append('--show-traceback') + args.append('--no-infer-executable') self.waiter.add(LazySubprocess(full_name, args, cwd=cwd, env=self.env)) def add_mypy(self, name: str, *args: str, cwd: Optional[str] = None) -> None: From 1582692c54fb303e4580097cb0f50197e865d295 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 7 Mar 2018 00:24:18 -0800 Subject: [PATCH 6/6] Add test for python-executable and no-infer-executable --- mypy/test/testargs.py | 51 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/mypy/test/testargs.py b/mypy/test/testargs.py index 20db610cda18..337b591fb0dd 100644 --- a/mypy/test/testargs.py +++ b/mypy/test/testargs.py @@ -4,10 +4,15 @@ defaults, and that argparse doesn't assign any new members to the Options object it creates. """ +import argparse +import sys + +import pytest # type: ignore from mypy.test.helpers import Suite, assert_equal from mypy.options import Options -from mypy.main import process_options +from mypy.main import (process_options, PythonExecutableInferenceError, + infer_python_version_and_executable) class ArgSuite(Suite): @@ -17,3 +22,47 @@ def test_coherence(self) -> None: # FIX: test this too. Requires changing working dir to avoid finding 'setup.cfg' options.config_file = parsed_options.config_file assert_equal(options, parsed_options) + + def test_executable_inference(self) -> None: + """Test the --python-executable flag with --python-version""" + sys_ver_str = '.'.join(map(str, sys.version_info[:2])) + + base = ['file.py'] # dummy file + + # test inference given one (infer the other) + matching_version = base + ['--python-version={}'.format(sys_ver_str)] + _, options = process_options(matching_version) + assert options.python_version == sys.version_info[:2] + assert options.python_executable == sys.executable + + matching_version = base + ['--python-executable={}'.format(sys.executable)] + _, options = process_options(matching_version) + assert options.python_version == sys.version_info[:2] + assert options.python_executable == sys.executable + + # test inference given both + matching_version = base + ['--python-version={}'.format(sys_ver_str), + '--python-executable={}'.format(sys.executable)] + _, options = process_options(matching_version) + assert options.python_version == sys.version_info[:2] + assert options.python_executable == sys.executable + + # test that we error if the version mismatch + # argparse sys.exits on a parser.error, we need to check the raw inference function + options = Options() + + special_opts = argparse.Namespace() + special_opts.python_executable = sys.executable + special_opts.python_version = (2, 10) # obviously wrong + special_opts.no_executable = None + with pytest.raises(PythonExecutableInferenceError) as e: + options = infer_python_version_and_executable(options, special_opts) + assert str(e.value) == 'Python version (2, 10) did not match executable {}, got' \ + ' version {}.'.format(sys.executable, str(sys.version_info[:2])) + + # test that --no-infer-executable will disable executable inference + matching_version = base + ['--python-version={}'.format(sys_ver_str), + '--no-infer-executable'] + _, options = process_options(matching_version) + assert options.python_version == sys.version_info[:2] + assert options.python_executable is None