scikit-learn · maskani-moh · Aug 2, 2017 · Aug 2, 2017 · Aug 3, 2017 · Aug 4, 2017
diff --git a/build_tools/travis/install.sh b/build_tools/travis/install.sh
@@ -39,6 +39,12 @@ if [[ "$DISTRIB" == "conda" ]]; then
 
     # Configure the conda environment and put it in the path using the
     # provided versions
+    if [[ "$USE_PYTEST" == "true" ]]; then
+        TEST_RUNNER_PACKAGE=pytest
+    else
+        TEST_RUNNER_PACKAGE=nose
+    fi
+
     if [[ "$INSTALL_MKL" == "true" ]]; then
         conda create -n testenv --yes python=$PYTHON_VERSION pip \
             pytest pytest-cov numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION \

diff --git a/doc/README.md b/doc/README.md
@@ -1,5 +1,6 @@
 # Documentation for scikit-learn
 
+
 This directory contains the full manual and web site as displayed at
 http://scikit-learn.org. See
 http://scikit-learn.org/dev/developers/contributing.html#documentation for

diff --git a/doc/datasets/conftest.py b/doc/datasets/conftest.py
@@ -0,0 +1,75 @@
+from os.path import exists
+from os.path import join
+
+import numpy as np
+
+from sklearn.utils.testing import SkipTest
+from sklearn.utils.testing import check_skip_network
+from sklearn.datasets import get_data_home
+from sklearn.utils.testing import install_mldata_mock
+from sklearn.utils.testing import uninstall_mldata_mock
+
+
+def setup_labeled_faces():
+    data_home = get_data_home()
+    if not exists(join(data_home, 'lfw_home')):
+        raise SkipTest("Skipping dataset loading doctests")
+
+
+def setup_mldata():
+    # setup mock urllib2 module to avoid downloading from mldata.org
+    install_mldata_mock({
+        'mnist-original': {
+            'data': np.empty((70000, 784)),
+            'label': np.repeat(np.arange(10, dtype='d'), 7000),
+        },
+        'iris': {
+            'data': np.empty((150, 4)),
+        },
+        'datasets-uci-iris': {
+            'double0': np.empty((150, 4)),
+            'class': np.empty((150,)),
+        },
+    })
+
+
+def teardown_mldata():
+    uninstall_mldata_mock()
+
+
+def setup_rcv1():
+    check_skip_network()
+    # skip the test in rcv1.rst if the dataset is not already loaded
+    rcv1_dir = join(get_data_home(), "RCV1")
+    if not exists(rcv1_dir):
+        raise SkipTest("Download RCV1 dataset to run this test.")
+
+
+def setup_twenty_newsgroups():
+    data_home = get_data_home()
+    if not exists(join(data_home, '20news_home')):
+        raise SkipTest("Skipping dataset loading doctests")
+
+
+def setup_working_with_text_data():
+    check_skip_network()
+
+
+def pytest_runtest_setup(item):
+    fname = item.fspath.strpath
+    if fname.endswith('datasets/labeled_faces.rst'):
+        setup_labeled_faces()
+    elif fname.endswith('datasets/mldata.rst'):
+        setup_mldata()
+    elif fname.endswith('datasets/rcv1.rst'):
+        setup_rcv1()
+    elif fname.endswith('datasets/twenty_newsgroups.rst'):
+        setup_twenty_newsgroups()
+    elif fname.endswith('datasets/working_with_text_data.rst'):
+        setup_working_with_text_data()
+
+
+def pytest_runtest_teardown(item):
+    fname = item.fspath.strpath
+    if fname.endswith('datasets/mldata.rst'):
+        teardown_mldata()
diff --git a/doc/datasets/mldata_fixture.py b/doc/datasets/mldata_fixture.py
@@ -0,0 +1,30 @@
+"""Fixture module to skip the datasets loading when offline
+
+Mock urllib2 access to mldata.org and create a temporary data folder.
+"""
+
+import numpy as np
+
+from sklearn.utils.testing import install_mldata_mock
+from sklearn.utils.testing import uninstall_mldata_mock
+
+
+def setup_module():
+    # setup mock urllib2 module to avoid downloading from mldata.org
+    install_mldata_mock({
+        'mnist-original': {
+            'data': np.empty((70000, 784)),
+            'label': np.repeat(np.arange(10, dtype='d'), 7000),
+        },
+        'iris': {
+            'data': np.empty((150, 4)),
+        },
+        'datasets-uci-iris': {
+            'double0': np.empty((150, 4)),
+            'class': np.empty((150,)),
+        },
+    })
+
+
+def teardown_module():
+    uninstall_mldata_mock()
diff --git a/doc/modules/clustering.rst b/doc/modules/clustering.rst
@@ -1150,7 +1150,7 @@ With :math:`P'(j) = |V_j| / N`. The mutual information (MI) between :math:`U`
 and :math:`V` is calculated by:
 
 .. math:: \text{MI}(U, V) = \sum_{i=1}^{|U|}\sum_{j=1}^{|V|}P(i, j)\log\left(\frac{P(i,j)}{P(i)P'(j)}\right)
-
+ 
 where :math:`P(i, j) = |U_i \cap V_j| / N` is the probability that an object
 picked at random falls into both classes :math:`U_i` and :math:`V_j`.
 

diff --git a/doc/modules/model_evaluation.rst b/doc/modules/model_evaluation.rst
@@ -309,15 +309,16 @@ Some also work in the multilabel case:
    precision_recall_fscore_support
    precision_score
    recall_score
+   roc_auc_score
    zero_one_loss
 
+
 And some work with binary and multilabel (but not multiclass) problems:
 
 .. autosummary::
    :template: function.rst
 
    average_precision_score
-   roc_auc_score
 
 
 In the following sub-sections, we will describe each of those functions,
@@ -1169,10 +1170,41 @@ In multi-label classification, the :func:`roc_auc_score` function is
 extended by averaging over the labels as :ref:`above <average>`.
 
 Compared to metrics such as the subset accuracy, the Hamming loss, or the
-F1 score, ROC doesn't require optimizing a threshold for each label. The
-:func:`roc_auc_score` function can also be used in multi-class classification,
-if the predicted outputs have been binarized.
+F1 score, ROC doesn't require optimizing a threshold for each label.
+
+The :func:`roc_auc_score` function can also be used in multi-class
+classification. [F2009]_ Two averaging strategies are currently supported: the
+one-vs-one algorithm computes the average of the pairwise
+ROC AUC scores, and the one-vs-rest algorithm
+computes the average of the ROC AUC scores for each class against
+all other classes. In both cases, the predicted class labels are provided in
+an array with values from 0 to ``n_classes``, and the scores correspond to the
+probability estimates that a sample belongs to a particular class.
+
+**One-vs-one Algorithm**
+The AUC of each class against each other, computing
+the AUC of all possible pairwise combinations :math:`c(c-1)` for a
+:math:`c`-dimensional classifier.
+
+[HT2001]_ Using the uniform class distribution:
+
+.. math:: \frac{1}{c(c-1)}\sum_{j=1}^c\sum_{k \neq j}^c \textnormal{AUC}(j, k)
+
+[F2009]_ Weighted by the prevalence of classes `j` and `k`:
+
+.. math:: \frac{1}{c-1}\sum_{j=1}^c\sum_{k \neq j}^c p(j \cup k)\textnormal{AUC}(j, k)
 
+**One-vs-rest Algorithm**
+AUC of each class against the rest. This treats
+a :math:`c`-dimensional classifier as :math:`c` two-dimensional classifiers.
+
+[F2006]_ Using the uniform class distribution:
+
+.. math:: \frac{\sum_{j=1}^c \textnormal{AUC}(j, \textnormal{rest}_j)}{c}
+
+[F2001]_ Weighted by the a priori class distribution:
+
+.. math:: \frac{\sum_{j=1}^c p(j)\textnormal{AUC}(j, \textnormal{rest}_j)}{c}
 
 .. image:: ../auto_examples/model_selection/images/sphx_glr_plot_roc_002.png
    :target: ../auto_examples/model_selection/plot_roc.html
@@ -1193,6 +1225,24 @@ if the predicted outputs have been binarized.
     for an example of using ROC to
     model species distribution.
 
+.. topic:: References:
+
+    .. [F2001] Fawcett, T., 2001. `Using rule sets to maximize 
+       ROC performance <http://ieeexplore.ieee.org/document/989510/>`_
+       In Data Mining, 2001.
+       Proceedings IEEE International Conference, pp. 131-138.
+    .. [F2006] Fawcett, T., 2006. `An introduction to ROC analysis.
+       <http://www.sciencedirect.com/science/article/pii/S016786550500303X>`_
+       Pattern Recognition Letters, 27(8), pp. 861-874.
+    .. [F2009] Ferri, C., Hernandez-Orallo, J., and Modroiu, R., 2009.
+       `An experimental comparison of performance measures for classification.
+       <http://www.sciencedirect.com/science/article/pii/S0167865508002687>`_
+       Pattern Recognition Letters, 30(1), pp. 27-38.
+    .. [HT2001] Hand, D.J. and Till, R.J., 2001. `A simple generalisation
+       of the area under the ROC curve for multiple class classification problems.
+       <http://link.springer.com/article/10.1023/A:1010920819831>`_
+       Machine learning, 45(2), pp.171-186.
+
 .. _zero_one_loss:
 
 Zero one loss

diff --git a/doc/modules/multiclass.rst b/doc/modules/multiclass.rst
@@ -436,4 +436,4 @@ Regressor Chain
 Regressor chains (see :class:`RegressorChain`) is analogous to 
 ClassifierChain as a way of combining a number of regressions 
 into a single multi-target model that is capable of exploiting 
-correlations among targets.
+correlations among targets.
diff --git a/doc/modules/preprocessing.rst b/doc/modules/preprocessing.rst
@@ -302,6 +302,7 @@ This can be confirmed on a independent testing set with similar remarks::
   ... # doctest: +ELLIPSIS +SKIP
   array([ 0.01...,  0.25...,  0.46...,  0.60... ,  0.94...])
 
+
 Mapping to a Gaussian distribution
 ----------------------------------
 

diff --git a/doc/whats_new/v0.20.rst b/doc/whats_new/v0.20.rst
@@ -67,6 +67,7 @@ Preprocessing
   :issue:`10210` by :user:`Eric Chang <ericchang00>` and
   :user:`Maniteja Nandana <maniteja123>`.
 
+>>>>>>> upstream/master
 
 Model evaluation
-Original file line number
+Diff line change
@@ Expand Up @@
       ... # doctest: +ELLIPSIS +SKIP
       array([ 0.01...,  0.25...,  0.46...,  0.60... ,  0.94...])
     Mapping to a Gaussian distribution
     ----------------------------------
@@ Expand Down @@