aimacode · delaray · Sep 1, 2019 · Jul 26, 2019 · Jul 28, 2019 · Aug 5, 2019
diff --git a/DeepNeuralNet4e.py b/DeepNeuralNet4e.py
@@ -209,7 +209,7 @@ def init_examples(examples, idx_i, idx_t, o_units):
 # 19.4.1 Stochastic gradient descent
 
 
-def gradient_descent(dataset, net, loss, epochs=1000, l_rate=0.01,  batch_size=1):
+def gradient_descent(dataset, net, loss, epochs=1000, l_rate=0.01,  batch_size=1, verbose=None):
     """
     gradient descent algorithm to update the learnable parameters of a network.
     :return: the updated network.
@@ -236,15 +236,15 @@ def gradient_descent(dataset, net, loss, epochs=1000, l_rate=0.01,  batch_size=1
                     for j in range(len(weights[i])):
                         net[i].nodes[j].weights = weights[i][j]
 
-        if (e+1) % 10 == 0:
+        if verbose and (e+1) % verbose == 0:
             print("epoch:{}, total_loss:{}".format(e+1,total_loss))
     return net
 
 
 # 19.4.2 Other gradient-based optimization algorithms
 
 
-def adam_optimizer(dataset, net,  loss, epochs=1000, rho=(0.9, 0.999), delta=1/10**8, l_rate=0.001, batch_size=1):
+def adam_optimizer(dataset, net,  loss, epochs=1000, rho=(0.9, 0.999), delta=1/10**8, l_rate=0.001, batch_size=1, verbose=None):
     """
     Adam optimizer in Figure 19.6 to update the learnable parameters of a network.
     Required parameters are similar to gradient descent.
@@ -288,7 +288,7 @@ def adam_optimizer(dataset, net,  loss, epochs=1000, rho=(0.9, 0.999), delta=1/1
                     for j in range(len(weights[i])):
                         net[i].nodes[j].weights = weights[i][j]
 
-        if (e+1) % 10 == 0:
+        if verbose and (e+1) % verbose == 0:
             print("epoch:{}, total_loss:{}".format(e+1,total_loss))
     return net
 
@@ -382,7 +382,7 @@ def get_batch(examples, batch_size=1):
 # example of NNs
 
 
-def neural_net_learner(dataset, hidden_layer_sizes=[4], learning_rate=0.01, epochs=100, optimizer=gradient_descent, batch_size=1):
+def neural_net_learner(dataset, hidden_layer_sizes=[4], learning_rate=0.01, epochs=100, optimizer=gradient_descent, batch_size=1, verbose=None):
     """Example of a simple dense multilayer neural network.
     :param hidden_layer_sizes: size of hidden layers in the form of a list"""
 
@@ -399,7 +399,7 @@ def neural_net_learner(dataset, hidden_layer_sizes=[4], learning_rate=0.01, epoc
     raw_net.append(DenseLayer(hidden_input_size, output_size))
 
     # update parameters of the network
-    learned_net = optimizer(dataset, raw_net, mse_loss, epochs, l_rate=learning_rate, batch_size=batch_size)
+    learned_net = optimizer(dataset, raw_net, mse_loss, epochs, l_rate=learning_rate, batch_size=batch_size, verbose=verbose)
 
     def predict(example):
         n_layers = len(learned_net)
@@ -417,7 +417,7 @@ def predict(example):
     return predict
 
 
-def perceptron_learner(dataset, learning_rate=0.01, epochs=100):
+def perceptron_learner(dataset, learning_rate=0.01, epochs=100, verbose=None):
     """
     Example of a simple perceptron neural network.
     """
@@ -427,7 +427,7 @@ def perceptron_learner(dataset, learning_rate=0.01, epochs=100):
     # initialize the network, add dense layer
     raw_net = [InputLayer(input_size), DenseLayer(input_size, output_size)]
     # update the network
-    learned_net = gradient_descent(dataset, raw_net, mse_loss, epochs, l_rate=learning_rate)
+    learned_net = gradient_descent(dataset, raw_net, mse_loss, epochs, l_rate=learning_rate, verbose=verbose)
 
     def predict(example):
 

diff --git a/learning4e.py b/learning4e.py
@@ -1,6 +1,6 @@
 from utils4e import (
     removeall, unique, mode, argmax_random_tie, isclose, dotproduct, weighted_sample_with_replacement,
-    num_or_str, normalize, clip, print_table, open_data, probability, random_weights
+    num_or_str, normalize, clip, print_table, open_data, probability, random_weights, euclidean_distance
 )
 
 import copy
@@ -382,8 +382,8 @@ def cross_validation(learner, size, dataset, k=10, trials=1):
         examples = dataset.examples
         random.shuffle(dataset.examples)
         for fold in range(k):
-            train_data, val_data = train_test_split(dataset, fold * (n / k),
-                                                    (fold + 1) * (n / k))
+            train_data, val_data = train_test_split(dataset, fold * (n // k),
+                                                    (fold + 1) * (n // k))
             dataset.examples = train_data
             h = learner(dataset, size)
             fold_errs += err_ratio(h, dataset, train_data)
@@ -393,6 +393,37 @@ def cross_validation(learner, size, dataset, k=10, trials=1):
         return fold_errs/k
 
 
+def cross_validation_nosize(learner, dataset, k=10, trials=1):
+    """Do k-fold cross_validate and return their mean.
+    That is, keep out 1/k of the examples for testing on each of k runs.
+    Shuffle the examples first; if trials>1, average over several shuffles.
+    Returns Training error, Validataion error"""
+    k = k or len(dataset.examples)
+    if trials > 1:
+        trial_errs = 0
+        for t in range(trials):
+            errs = cross_validation(learner, dataset,
+                                          k=10, trials=1)
+            trial_errs += errs
+        return trial_errs/trials
+    else:
+        fold_errs = 0
+        n = len(dataset.examples)
+        examples = dataset.examples
+        random.shuffle(dataset.examples)
+        for fold in range(k):
+            train_data, val_data = train_test_split(dataset, fold * (n // k),
+                                                    (fold + 1) * (n // k))
+            dataset.examples = train_data
+            h = learner(dataset)
+            fold_errs += err_ratio(h, dataset, train_data)
+
+            # Reverting back to original once test is completed
+            dataset.examples = examples
+        return fold_errs/k
+
+
+
 def err_ratio(predict, dataset, examples=None, verbose=0):
     """Return the proportion of the examples that are NOT correctly predicted.
     verbose - 0: No output; 1: Output wrong; 2 (or greater): Output correct"""
@@ -521,6 +552,8 @@ def LinearLearner(dataset, learning_rate=0.01, epochs=100):
         for example in examples:
             x = [1] + example
             y = dotproduct(w, x)
+            # if threshold:
+            #     y = threshold(y)
             t = example[idx_t]
             err.append(t - y)
 
@@ -554,17 +587,20 @@ def LogisticLinearLeaner(dataset, learning_rate=0.01, epochs=100):
 
     for epoch in range(epochs):
         err = []
+        h= []
         # Pass over all examples
         for example in examples:
             x = [1] + example
             y = 1/(1 + math.exp(-dotproduct(w, x)))
-            h = [y * (1-y)]
+            h.append(y * (1-y))
             t = example[idx_t]
             err.append(t - y)
 
         # update weights
         for i in range(len(w)):
-            w[i] = w[i] + learning_rate * (dotproduct(dotproduct(err,h), X_col[i]) / num_examples)
+            buffer = [x*y for x,y in zip(err, h)]
+            # w[i] = w[i] + learning_rate * (dotproduct(err, X_col[i]) / num_examples)
+            w[i] = w[i] + learning_rate * (dotproduct(buffer, X_col[i]) / num_examples)
 
     def predict(example):
         x = [1] + example
@@ -580,6 +616,7 @@ def NearestNeighborLearner(dataset, k=1):
     """k-NearestNeighbor: the k nearest neighbors vote."""
     def predict(example):
         """Find the k closest items, and have them vote for the best."""
+        example.pop(dataset.target)
         best = heapq.nsmallest(k, ((dataset.distance(e, example), e)
                                    for e in dataset.examples))
         return mode(e[dataset.target] for (d, e) in best)
@@ -829,6 +866,6 @@ def compare(algorithms=None, datasets=None, k=10, trials=1):
                             Majority(7, 100), Parity(7, 100), Xor(100)]              # of datasets
 
     print_table([[a.__name__.replace('Learner', '')] +
-                 [cross_validation(a, d, k, trials) for d in datasets]
+                 [cross_validation_nosize(a, d, k, trials) for d in datasets]
                  for a in algorithms],
-                header=[''] + [d.name[0:7] for d in datasets], numfmt='%.2f')
+                header=[''] + [d.name[0:7] for d in datasets], numfmt='{0:.2f}')