Add support for generating code coverege reports (#6694)

gaborbernat · msullivan · commit ebb6cb867315 · 2019-04-26T13:13:40.000-07:00
diff --git a/.travis.yml b/.travis.yml
@@ -63,7 +63,7 @@ jobs:
 
 install:
 - pip install -U pip setuptools
-- pip install -U tox==3.7.0
+- pip install -U tox==3.9.0
 - tox --notest
 
 script:
diff --git a/appveyor.yml b/appveyor.yml
@@ -10,7 +10,7 @@ environment:
 
 install:
     - "git submodule update --init mypy/typeshed"
-    - "%PYTHON%\\python.exe -m pip install -U setuptools tox==3.7.0"
+    - "%PYTHON%\\python.exe -m pip install -U setuptools tox==3.9.0"
     - "%PYTHON%\\python.exe -m tox -e py37 --notest"
 
 build: off
diff --git a/setup.cfg b/setup.cfg
@@ -52,3 +52,17 @@ parallel = true
 
 [coverage:report]
 show_missing = true
+skip_covered = True
+omit = mypy/test/*
+exclude_lines =
+    \#\s*pragma: no cover
+    ^\s*raise AssertionError\b
+    ^\s*raise NotImplementedError\b
+    ^\s*return NotImplemented\b
+    ^\s*raise$
+    ^if __name__ == ['"]__main__['"]:$
+
+[coverage:paths]
+source = mypy
+         .tox/*/lib/python*/site-packages/mypy
+         .tox\*\Lib\site-packages\mypy
diff --git a/tox.ini b/tox.ini
@@ -1,22 +1,45 @@
 [tox]
-minversion = 3.5.1
+minversion = 3.8.0
 skip_missing_interpreters = true
-envlist = py35,
-          py36,
-          py37,
-          lint,
-          type,
-          docs
+envlist =
+    py35,
+    py36,
+    py37,
+    lint,
+    type,
+    docs,
 isolated_build = true
 
 [testenv]
 description = run the test driver with {basepython}
+setenv = cov: COVERAGE_FILE={toxworkdir}/.coverage.{envname}
 passenv = PYTEST_XDIST_WORKER_COUNT NEWSEMANAL
 deps = -rtest-requirements.txt
-commands = pytest {posargs}
+commands = python -m pytest {posargs}
+           cov: python -m pytest {posargs: --cov mypy --cov-config setup.cfg}
+    
 
-[testenv:.package]
+[testenv:coverage]
+description = [run locally after tests]: combine coverage data and create report
 deps =
+    coverage >= 4.5.1, < 5
+    diff_cover >= 1.0.5, <2
+skip_install = True
+passenv =
+    {[testenv]passenv}
+    DIFF_AGAINST
+setenv = COVERAGE_FILE={toxworkdir}/.coverage
+commands =
+    coverage combine --rcfile setup.cfg
+    coverage report -m --rcfile setup.cfg
+    coverage xml -o {toxworkdir}/coverage.xml --rcfile setup.cfg
+    coverage html -d {toxworkdir}/htmlcov --rcfile setup.cfg
+    diff-cover --compare-branch {env:DIFF_AGAINST:origin/master} {toxworkdir}/coverage.xml
+depends =
+    py35,
+    py36,
+    py37,
+parallel_show_output = True
 
 [testenv:lint]
 description = check the code style
@@ -26,19 +49,22 @@ commands = flake8 -j0 {posargs}
 [testenv:type]
 description = type check ourselves
 basepython = python3.7
-commands = python3 -m mypy --config-file mypy_self_check.ini -p mypy
+commands = python -m mypy --config-file mypy_self_check.ini -p mypy
 
 [testenv:docs]
 description = invoke sphinx-build to build the HTML docs
 basepython = python3.7
 deps = -rdocs/requirements-docs.txt
-commands = sphinx-build -d "{toxworkdir}/docs_doctree" docs/source "{toxworkdir}/docs_out" --color -W -bhtml {posargs}
+commands =
+    sphinx-build -d "{toxworkdir}/docs_doctree" docs/source "{toxworkdir}/docs_out" --color -W -bhtml {posargs}
+    python -c 'import pathlib; print("documentation available under file://\{0\}".format(pathlib.Path(r"{toxworkdir}") / "docs_out" / "index.html"))'
 
 [testenv:dev]
 description = generate a DEV environment, that has all project libraries
 usedevelop = True
-basepython = python3.7
-deps = -rtest-requirements.txt
-       -rdocs/requirements-docs.txt
-commands = python -m pip list --format=columns
-           python -c 'import sys; print(sys.executable)'
+deps =
+    -rtest-requirements.txt
+    -rdocs/requirements-docs.txt
+commands =
+    python -m pip list --format=columns
+    python -c 'import sys; print(sys.executable)'
