From 515c3c0cf9aa045b6cdb8b520fe7e6eff294c547 Mon Sep 17 00:00:00 2001 From: Andrew Seier Date: Fri, 30 Dec 2016 15:06:47 -0800 Subject: [PATCH] Try tox. --- .gitignore | 4 +- circle.yml | 35 +++++++-------- circle/setup.sh | 44 ------------------- circle/test.sh | 47 --------------------- contributing.md | 48 ++++++++++++++++++++- tox.ini | 110 ++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 177 insertions(+), 111 deletions(-) delete mode 100644 circle/setup.sh delete mode 100644 circle/test.sh create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index a8fd07363f4..6e19e2c2216 100644 --- a/.gitignore +++ b/.gitignore @@ -8,10 +8,12 @@ *.coverage +*.tox + build dist debug_script.py -test_output.txt \ No newline at end of file +test_output.txt diff --git a/circle.yml b/circle.yml index 3c146d51ecc..30ae291017a 100644 --- a/circle.yml +++ b/circle.yml @@ -2,21 +2,20 @@ machine: environment: - PLOTLY_PACKAGE_ROOT: /home/ubuntu/${CIRCLE_PROJECT_REPONAME} PLOTLY_CONFIG_DIR: ${HOME}/.plotly - PLOTLY_PYTHON_VERSIONS: 2.7.8 3.3.3 3.4.1 - PLOTLY_CORE_REQUIREMENTS_FILE: ${PLOTLY_PACKAGE_ROOT}/requirements.txt - PLOTLY_OPTIONAL_REQUIREMENTS_FILE: ${PLOTLY_PACKAGE_ROOT}/optional-requirements.txt + TOX_TESTENV_PASSENV: PLOTLY_TOX_* + PLOTLY_TOX_PYTHON_27: /home/ubuntu/.pyenv/versions/2.7.10/bin/python2.7 + PLOTLY_TOX_PYTHON_33: /home/ubuntu/.pyenv/versions/3.3.3/bin/python3.3 + PLOTLY_TOX_PYTHON_34: /home/ubuntu/.pyenv/versions/3.4.3/bin/python3.4 + PLOTLY_TOX_PYTHON_35: /home/ubuntu/.pyenv/versions/3.5.0/bin/python3.5 dependencies: override: - # run all the pre-written installers (this will take a *while*) - - bash circle/setup.sh - - # install testing tools for circle's version of things - - pip install nose coverage + # install everything tox knows about and the plotly package. + - pip install tox + - tox --notest - pip install -I . # we need to cd out of the project root to ensure the install worked @@ -24,14 +23,21 @@ dependencies: cache_directories: - - "~/.pyenv/versions" # attempt to just cache installed pyenv things + # cache everything that tox installs for us. + - .tox test: override: - # run test suite in all our python versions - - bash circle/test.sh + # let tox do its thing. matplotlib is broken at the moment. + - tox -- -a '!matplotlib' + + # we setup tox to create coverage reports, move them to artifacts. + - mv ~/${CIRCLE_PROJECT_REPONAME}/.tox/coverage-reports ${CIRCLE_ARTIFACTS} + + # import it once normally (ensures .plotly exists) + - python -c "import plotly" # test that it imports when you don't have write permissions - sudo chmod -R 444 ${PLOTLY_CONFIG_DIR} && python -c "import plotly" @@ -39,8 +45,3 @@ test: # test that giving back write permissions works again # this also has to pass the test suite that follows - sudo chmod -R 777 ${PLOTLY_CONFIG_DIR} && python -c "import plotly" - - # test core things in the general 2.7 version that circle has - - nosetests -x plotly/tests/test_core --with-coverage --cover-package=plotly - - mkdir "${CIRCLE_ARTIFACTS}/2.7" || true - - coverage html -d "${CIRCLE_ARTIFACTS}/2.7" --title=2.7 diff --git a/circle/setup.sh b/circle/setup.sh deleted file mode 100644 index b659e83cea0..00000000000 --- a/circle/setup.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -SIG="☁☀☂" - -echo "${SIG} Running setup routine with Python versions:" -for version in ${PLOTLY_PYTHON_VERSIONS[@]}; do - echo "${SIG} ${version}" -done - -PROGNAME=$(basename $0) -function error_exit -{ - echo -e "${SIG} ${PROGNAME}: ${1:-"Unknown Error"}\n" 1>&2 - exit 1 -} - -# PYENV shims need to be infront of the rest of the path to work! -echo "${SIG} Adding pyenv shims to the beginning of the path in this shell." -export PATH="/home/ubuntu/.pyenv/shims:$PATH" - -# for each version we want, setup a functional virtual environment -for version in ${PLOTLY_PYTHON_VERSIONS[@]}; do - echo "${SIG} Setting up Python ${version}" - - # exporting this variable (in this scope) chooses the python version - export PYENV_VERSION=${version} - echo "${SIG} Using pyenv version $(pyenv version)" - - # this was a major issue previously, sanity check that we're using the - # version we *think* we're using (that pyenv is pointing to) - echo "${SIG} python -c 'import sys; print(sys.version)'" - python -c 'import sys; print(sys.version)' - - # install core requirements all versions need - pip install -r ${PLOTLY_CORE_REQUIREMENTS_FILE} || - error_exit "${SIG} ${LINENO}: can't install core reqs for Python ${version}." - - pip install -r ${PLOTLY_OPTIONAL_REQUIREMENTS_FILE} || - error_exit "${SIG} ${LINENO}: can't install optional for Python ${version}." - - # install some test tools - pip install nose coverage || - error_exit "${SIG} ${LINENO}: can't install test tools for Python ${version}." -done diff --git a/circle/test.sh b/circle/test.sh deleted file mode 100644 index 9f5c4451e39..00000000000 --- a/circle/test.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -SIG="☁☀☂" - -echo "${SIG} Running test routine with python versions:" -for version in ${PLOTLY_PYTHON_VERSIONS[@]}; do - echo "${SIG} ${version}" -done - -PROGNAME=$(basename $0) -function error_exit -{ - echo -e "${SIG} ${PROGNAME}: ${1:-"Unknown Error"}\n" 1>&2 - exit 1 -} - -# PYENV shims need to be infront of the rest of the path to work! -echo "${SIG} Adding pyenv shims to the beginning of the path in this shell." -export PATH="/home/ubuntu/.pyenv/shims:$PATH" - -# for each version we want, setup a functional virtual environment -for version in ${PLOTLY_PYTHON_VERSIONS[@]}; do - echo "${SIG} Testing Python ${version}" - - # exporting this variable (in this scope) chooses the python version - export PYENV_VERSION=${version} - echo "${SIG} Using pyenv version $(pyenv version)." - - # this was a major issue previously, sanity check that we're using the - # version we *think* we're using (that pyenv is pointing to) - echo "${SIG} Running: python -c 'import sys; print(sys.version)'. We've got:" - python -c 'import sys; print(sys.version)' - - - echo "${SIG} Install plotly (ignoring possibly cached versions)." - pip install -I ${PLOTLY_PACKAGE_ROOT} >/dev/null || - error_exit "${SIG} ${LINENO}: can't install plotly package from project root" - - echo "${SIG} Import plotly to create .plotly dir if DNE." - python -c 'import plotly' >/dev/null || - error_exit "${SIG} ${LINENO}: can't import plotly package" - - echo "${SIG} Running tests for Python ${version} as user '$(whoami)'." - nosetests -x -a '!matplotlib' plotly/tests || - error_exit "${SIG} ${LINENO}: test suite failed for Python ${version}" - -done diff --git a/contributing.md b/contributing.md index e25807abdce..5e052a7cd87 100644 --- a/contributing.md +++ b/contributing.md @@ -100,9 +100,12 @@ pip install -r optional-requirements.txt ##Testing -Our API uses Nose to run tests. (https://nose.readthedocs.org/en/latest/) +We take advantage of two tools to run tests: -###Running Tests +* [`tox`](https://tox.readthedocs.io/en/latest/), which is both a virtualenv management and test tool. +* [`nose`](https://nose.readthedocs.org/en/latest/), which is is an extension of Python's unittest + +###Running Tests with `nose` Since our tests cover *all* the functionality, to prevent tons of errors from showing up and having to parse through a messy output, you'll need to install `optional-requirements.txt` as explained above. @@ -130,6 +133,47 @@ nosetests -w plotly/tests/test_plotly nosetests plotly/tests/test_plotly/test_plot.py ``` +###Running tests with `tox` + +Running tests with tox is much more powerful, but requires a bit more setup. + +You'll need to export an environment variable for *each* tox environment you wish to test with. For example, if you want to test with `Python 2.7` and +`Python 3.4`, but only care to check the `core` specs, you would need to ensure that the following variables are exported: + +``` +export PLOTLY_TOX_PYTHON_27= +export PLOTLY_TOX_PYTHON_34= +``` + +Where the `: dep syntax. +; Only one dep is allowed per line as of the time of writing. The +; can be a `-` (hyphen) concatenated string of the environments to target +; with the given dep. + +; These commands are general and will be run for *all* environments. +[testenv] +passenv=PLOTLY_TOX_* +whitelist_externals= + mkdir +deps= + coverage==4.3.1 + mock==2.0.0 + nose==1.3.7 + requests==2.12.4 + six==1.10.0 + pytz==2016.10 + optional: numpy==1.11.3 + optional: ipython[all]==5.1.0 + optional: pandas==0.19.2 + optional: scipy==0.18.1 + +; CORE ENVIRONMENTS +[testenv:py27-core] +basepython={env:PLOTLY_TOX_PYTHON_27:} +commands= + python --version + nosetests {posargs} -x plotly/tests/test_core + +[testenv:py33-core] +basepython={env:PLOTLY_TOX_PYTHON_33:} +commands= + python --version + nosetests {posargs} -x plotly/tests/test_core + +[testenv:py34-core] +basepython={env:PLOTLY_TOX_PYTHON_34:} +commands= + python --version + nosetests {posargs} -x plotly/tests/test_core + +[testenv:py35-core] +basepython={env:PLOTLY_TOX_PYTHON_35:} +commands= + python --version + nosetests {posargs} -x plotly/tests/test_core + +; OPTIONAL ENVIRONMENTS +[testenv:py27-optional] +basepython={env:PLOTLY_TOX_PYTHON_27:} +commands= + python --version +; Do some coverage reporting. No need to do this for all environments. + mkdir -p {envbindir}/../../coverage-reports/{envname} + coverage erase + coverage run --include="*/plotly/*" --omit="*/tests*" {envbindir}/nosetests {posargs} -x plotly/tests + coverage html -d "{envbindir}/../../coverage-reports/{envname}" --title={envname} + +[testenv:py34-optional] +basepython={env:PLOTLY_TOX_PYTHON_34:} +commands= + python --version + nosetests {posargs} -x plotly/tests + +[testenv:py35-optional] +basepython={env:PLOTLY_TOX_PYTHON_35:} +commands= + python --version + nosetests {posargs} -x plotly/tests