plotly · theengineear · Jan 1, 2017 · Dec 30, 2016
diff --git a/.gitignore b/.gitignore
@@ -8,10 +8,12 @@
 
 *.coverage
 
+*.tox
+
 build
 
 dist
 
 debug_script.py
 
-test_output.txt
+test_output.txt
diff --git a/circle.yml b/circle.yml
@@ -2,45 +2,46 @@ machine:
 
   environment:
 
-    PLOTLY_PACKAGE_ROOT: /home/ubuntu/${CIRCLE_PROJECT_REPONAME}
     PLOTLY_CONFIG_DIR: ${HOME}/.plotly
-    PLOTLY_PYTHON_VERSIONS: 2.7.8 3.3.3 3.4.1
-    PLOTLY_CORE_REQUIREMENTS_FILE: ${PLOTLY_PACKAGE_ROOT}/requirements.txt
-    PLOTLY_OPTIONAL_REQUIREMENTS_FILE: ${PLOTLY_PACKAGE_ROOT}/optional-requirements.txt
+    TOX_TESTENV_PASSENV: PLOTLY_TOX_*
+    PLOTLY_TOX_PYTHON_27: /home/ubuntu/.pyenv/versions/2.7.10/bin/python2.7
+    PLOTLY_TOX_PYTHON_33: /home/ubuntu/.pyenv/versions/3.3.3/bin/python3.3
+    PLOTLY_TOX_PYTHON_34: /home/ubuntu/.pyenv/versions/3.4.3/bin/python3.4
+    PLOTLY_TOX_PYTHON_35: /home/ubuntu/.pyenv/versions/3.5.0/bin/python3.5
 
 dependencies:
 
   override:
 
-    # run all the pre-written installers (this will take a *while*)
-    - bash circle/setup.sh
-
-    # install testing tools for circle's version of things
-    - pip install nose coverage
+    # install everything tox knows about and the plotly package.
+    - pip install tox
+    - tox --notest
     - pip install -I .
 
     # we need to cd out of the project root to ensure the install worked
     - cd ~ && python -c "import plotly"
 
   cache_directories:
 
-    - "~/.pyenv/versions"  # attempt to just cache installed pyenv things
+    # cache everything that tox installs for us.
+    - .tox
 
 test:
 
   override:
 
-    # run test suite in all our python versions
-    - bash circle/test.sh
+    # let tox do its thing. matplotlib is broken at the moment.
+    - tox -- -a '!matplotlib'
+
+    # we setup tox to create coverage reports, move them to artifacts.
+    - mv ~/${CIRCLE_PROJECT_REPONAME}/.tox/coverage-reports ${CIRCLE_ARTIFACTS}
+
+    # import it once normally (ensures .plotly exists)
+    - python -c "import plotly"
 
     # test that it imports when you don't have write permissions
     - sudo chmod -R 444 ${PLOTLY_CONFIG_DIR} && python -c "import plotly"
 
     # test that giving back write permissions works again
     # this also has to pass the test suite that follows
     - sudo chmod -R 777 ${PLOTLY_CONFIG_DIR} && python -c "import plotly"
-
-    # test core things in the general 2.7 version that circle has
-    - nosetests -x plotly/tests/test_core --with-coverage --cover-package=plotly
-    - mkdir "${CIRCLE_ARTIFACTS}/2.7" || true
-    - coverage html -d "${CIRCLE_ARTIFACTS}/2.7" --title=2.7
diff --git a/circle/setup.sh b/circle/setup.sh
diff --git a/circle/test.sh b/circle/test.sh
diff --git a/contributing.md b/contributing.md
@@ -100,9 +100,12 @@ pip install -r optional-requirements.txt
 
 ##Testing
 
-Our API uses Nose to run tests. (https://nose.readthedocs.org/en/latest/)
+We take advantage of two tools to run tests:
 
-###Running Tests
+* [`tox`](https://tox.readthedocs.io/en/latest/), which is both a virtualenv management and test tool.
+* [`nose`](https://nose.readthedocs.org/en/latest/), which is is an extension of Python's unittest
+
+###Running Tests with `nose`
 
 Since our tests cover *all* the functionality, to prevent tons of errors from showing up and having to parse through a messy output, you'll need to install `optional-requirements.txt` as explained above.
 
@@ -130,6 +133,47 @@ nosetests -w plotly/tests/test_plotly
 nosetests plotly/tests/test_plotly/test_plot.py
 ```
 
+###Running tests with `tox`
+
+Running tests with tox is much more powerful, but requires a bit more setup.
+
+You'll need to export an environment variable for *each* tox environment you wish to test with. For example, if you want to test with `Python 2.7` and
+`Python 3.4`, but only care to check the `core` specs, you would need to ensure that the following variables are exported:
+
+```
+export PLOTLY_TOX_PYTHON_27=<python binary>
+export PLOTLY_TOX_PYTHON_34=<python binary>
+```
+
+Where the `<python binary` is going to be specific to your development setup. As a more complete example, you might have this loaded in a `.bash_profile` (or equivalent shell loader):
+
+```bash
+############
+# tox envs #
+############
+
+export PLOTLY_TOX_PYTHON_27=python2.7
+export PLOTLY_TOX_PYTHON_34=python3.4
+export TOXENV=py27-core,py34-core
+```
+
+Where `TOXENV` is the environment list you want to use when invoking `tox` from the command line. Note that the `PLOTLY_TOX_*` pattern is used to pass in variables for use in the `tox.ini` file. Though this is a little setup, intensive, you'll get the following benefits:
+
+* `tox` will automatically manage a virtual env for each environment you want to test in.
+* You only have to run `tox` and know that the module is working in both `Python 2` and `Python 3`.
+
+Finally, `tox` allows you to pass in additional command line arguments that are formatted in (by us) in the `tox.ini` file, see `{posargs}`. This is setup to help with our `nose attr` configuration. To run only tests that are *not* tagged with `slow`, you could use the following command:
+
+```bash
+tox -- -a '!slow'
+```
+
+Note that anything after `--` is substituted in for `{posargs}` in the tox.ini. For completeness, because it's reasonably confusing, if you want to force a match for *multiple* `nose attr` tags, you comma-separate the tags like so:
+
+```bash
+tox -- -a '!slow','!matplotlib'
+```
+
 ###Writing Tests
 
 You're *strongly* encouraged to write tests that check your added functionality. 

diff --git a/tox.ini b/tox.ini
@@ -0,0 +1,110 @@
+; Tox is a testing tool that manages virtualenvs for testing multiple Python
+; environments in a consistent/controlled way.
+
+; SETTING ENVIRONMENT VARIABLES AND TOX TESTING VARIABLES
+;
+; You can limit tox testing to certain environments via the `-e` (envlist)
+; command line option:
+;    tox -e py27-core,py34-core
+; OR, you can just set the `TOXENV` environment variable, which is handy:
+;    TOXENV=py27-core,py34-core
+;
+; Integrating with the virtualenvs in Circle CI is a bit of a pain. For
+; whatever reason the "executable" `python35` (at the time of writing) cannot
+; be activated directly. Instead the circle.yml file specifies the actual
+; binary directly. Because of this, you too have to set the following env
+; variables:
+;    PLOTLY_TOX_PYTHON_27=python2.7
+;    PLOTLY_TOX_PYTHON_34=python3.4
+;    ...
+; These will be specific to your machine and may not look like the ones above.
+; If you're not testing with all the python versions (see TOXENV above),
+; there's no need to install and map other versions.
+
+; PASSING ADDITONAL ARGUMENTS TO TEST COMMANDS
+; The {posargs} is tox-specific and passes in any command line args after `--`.
+; For example, given the testing command in *this* file:
+;   nosetests {posargs} -x plotly/tests/test_core
+;
+; The following command:
+;   tox -- -a '!slow'
+;
+; Tells tox to call:
+;   nosetests -a '!slow' -x plotly/tests/test_core
+;
+; Which is a nice way to skip slow tests for faster testing cycles.
+
+[tox]
+; The py{A,B,C}-{X,Y} generates a matrix of envs:
+;   pyA-X,pyA-Y,pyB-X,pyB-Y,pyC-X,pyC-Y
+envlist = py{27,34,35}-{core,optional},py33-core
+
+; Note that envs can be targeted by deps using the <target>: dep syntax.
+; Only one dep is allowed per line as of the time of writing. The <target>
+; can be a `-` (hyphen) concatenated string of the environments to target
+; with the given dep.
+
+; These commands are general and will be run for *all* environments.
+[testenv]
+passenv=PLOTLY_TOX_*
+whitelist_externals=
+    mkdir
+deps=
+    coverage==4.3.1
+    mock==2.0.0
+    nose==1.3.7
+    requests==2.12.4
+    six==1.10.0
+    pytz==2016.10
+    optional: numpy==1.11.3
+    optional: ipython[all]==5.1.0
+    optional: pandas==0.19.2
+    optional: scipy==0.18.1
+
+; CORE ENVIRONMENTS
+[testenv:py27-core]
+basepython={env:PLOTLY_TOX_PYTHON_27:}
+commands=
+    python --version
+    nosetests {posargs} -x plotly/tests/test_core
+
+[testenv:py33-core]
+basepython={env:PLOTLY_TOX_PYTHON_33:}
+commands=
+    python --version
+    nosetests {posargs} -x plotly/tests/test_core
+
+[testenv:py34-core]
+basepython={env:PLOTLY_TOX_PYTHON_34:}
+commands=
+    python --version
+    nosetests {posargs} -x plotly/tests/test_core
+
+[testenv:py35-core]
+basepython={env:PLOTLY_TOX_PYTHON_35:}
+commands=
+    python --version
+    nosetests {posargs} -x plotly/tests/test_core
+
+; OPTIONAL ENVIRONMENTS
+[testenv:py27-optional]
+basepython={env:PLOTLY_TOX_PYTHON_27:}
+commands=
+    python --version
+;   Do some coverage reporting. No need to do this for all environments.
+    mkdir -p {envbindir}/../../coverage-reports/{envname}
+    coverage erase
+    coverage run --include="*/plotly/*" --omit="*/tests*" {envbindir}/nosetests {posargs} -x plotly/tests
+    coverage html -d "{envbindir}/../../coverage-reports/{envname}" --title={envname}
+
+[testenv:py34-optional]
+basepython={env:PLOTLY_TOX_PYTHON_34:}
+commands=
+    python --version
+    nosetests {posargs} -x plotly/tests
+
+[testenv:py35-optional]
+basepython={env:PLOTLY_TOX_PYTHON_35:}
+commands=
+    python --version
+    nosetests {posargs} -x plotly/tests