Learning: Grading Learners (aimacode#499)

antmarakis · norvig · commit 0879c4bf3a9d · 2017-04-17T14:05:49.000-07:00
* Update learning.py

* Update test_learning.py

* Update test_learning.py
diff --git a/learning.py b/learning.py
@@ -806,8 +806,9 @@ def flatten(seqs): return sum(seqs, [])
 # Functions for testing learners on examples
 
 
-def test(predict, dataset, examples=None, verbose=0):
+def err_ratio(predict, dataset, examples=None, verbose=0):
     """Return the proportion of the examples that are NOT correctly predicted."""
+    """verbose - 0: No output; 1: Output wrong; 2 (or greater): Output correct"""
     if examples is None:
         examples = dataset.examples
     if len(examples) == 0:
@@ -826,6 +827,12 @@ def test(predict, dataset, examples=None, verbose=0):
     return 1 - (right / len(examples))
 
 
+def grade_learner(predict, tests):
+    """Grades the given learner based on how many tests it passes.
+    tests is a list with each element in the form: (values, output)."""
+    return mean(int(predict(X) == y) for X, y in tests)
+
+
 def train_and_test(dataset, start, end):
     """Reserve dataset.examples[start:end] for test; train on the remainder."""
     start = int(start)
@@ -863,8 +870,8 @@ def cross_validation(learner, size, dataset, k=10, trials=1):
                                                   (fold + 1) * (n / k))
             dataset.examples = train_data
             h = learner(dataset, size)
-            fold_errT += test(h, dataset, train_data)
-            fold_errV += test(h, dataset, val_data)
+            fold_errT += err_ratio(h, dataset, train_data)
+            fold_errV += err_ratio(h, dataset, val_data)
             # Reverting back to original once test is completed
             dataset.examples = examples
         return fold_errT / k, fold_errV / k
@@ -908,16 +915,6 @@ def score(learner, size):
     return [(size, mean([score(learner, size) for t in range(trials)]))
             for size in sizes]
 
-
-def grade_learner(predict, tests):
-    """Grades the given learner based on how many tests it passes.
-    tests is a list with each element in the form: (values, output)."""
-    correct = 0
-    for t in tests:
-        if predict(t[0]) == t[1]:
-            correct += 1
-    return correct
-
 # ______________________________________________________________________________
 # The rest of this file gives datasets for machine learning problems.
 
diff --git a/tests/test_learning.py b/tests/test_learning.py
@@ -1,7 +1,7 @@
 from learning import parse_csv, weighted_mode, weighted_replicate, DataSet, \
                      PluralityLearner, NaiveBayesLearner, NearestNeighborLearner, \
                      NeuralNetLearner, PerceptronLearner, DecisionTreeLearner, \
-                     euclidean_distance, grade_learner
+                     euclidean_distance, grade_learner, err_ratio
 from utils import DataFile
 
 
@@ -76,10 +76,14 @@ def test_neural_network_learner():
 
     nNL = NeuralNetLearner(iris, [5], 0.15, 75)
     tests = [([5, 3, 1, 0.1], 0),
-             ([6, 3, 3, 1.5], 1),
-             ([7.5, 4, 6, 2], 2)]
+             ([5, 3.5, 1, 0], 0),
+             ([6, 3, 4, 1.1], 1),
+             ([6, 2, 3.5, 1], 1),
+             ([7.5, 4, 6, 2], 2),
+             ([7, 3, 6, 2.5], 2)]
 
-    assert grade_learner(nNL, tests) >= 2
+    assert grade_learner(nNL, tests) >= 2/3
+    assert err_ratio(nNL, iris) < 0.25
 
 
 def test_perceptron():
@@ -90,7 +94,11 @@ def test_perceptron():
 
     perceptron = PerceptronLearner(iris)
     tests = [([5, 3, 1, 0.1], 0),
+             ([5, 3.5, 1, 0], 0),
              ([6, 3, 4, 1.1], 1),
-             ([7.5, 4, 6, 2], 2)]
+             ([6, 2, 3.5, 1], 1),
+             ([7.5, 4, 6, 2], 2),
+             ([7, 3, 6, 2.5], 2)]
 
-    assert grade_learner(perceptron, tests) >= 2
+    assert grade_learner(perceptron, tests) > 1/2
+    assert err_ratio(perceptron, iris) < 0.4
