Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 483bf81

Browse files
bruceyoungsysudelaray
bruceyoungsysu
authored and
delaray
committed
Demo of chapter 19 (4th eidtion) (aimacode#1102)
* add demo of chapter 18 * add chapter 19 demo * rm chapter 18 part * modify learners.ipynb * modify learners.ipynb
1 parent e5204f6 commit 483bf81

19 files changed

+2916
-23
lines changed

‎DeepNeuralNet4e.py

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,7 @@ def init_examples(examples, idx_i, idx_t, o_units):
209209
# 19.4.1 Stochastic gradient descent
210210

211211

212-
def gradient_descent(dataset, net, loss, epochs=1000, l_rate=0.01, batch_size=1):
212+
def gradient_descent(dataset, net, loss, epochs=1000, l_rate=0.01, batch_size=1, verbose=None):
213213
"""
214214
gradient descent algorithm to update the learnable parameters of a network.
215215
:return: the updated network.
@@ -236,15 +236,15 @@ def gradient_descent(dataset, net, loss, epochs=1000, l_rate=0.01, batch_size=1
236236
for j in range(len(weights[i])):
237237
net[i].nodes[j].weights = weights[i][j]
238238

239-
if (e+1) % 10 == 0:
239+
if verbose and (e+1) % verbose == 0:
240240
print("epoch:{}, total_loss:{}".format(e+1,total_loss))
241241
return net
242242

243243

244244
# 19.4.2 Other gradient-based optimization algorithms
245245

246246

247-
def adam_optimizer(dataset, net, loss, epochs=1000, rho=(0.9, 0.999), delta=1/10**8, l_rate=0.001, batch_size=1):
247+
def adam_optimizer(dataset, net, loss, epochs=1000, rho=(0.9, 0.999), delta=1/10**8, l_rate=0.001, batch_size=1, verbose=None):
248248
"""
249249
Adam optimizer in Figure 19.6 to update the learnable parameters of a network.
250250
Required parameters are similar to gradient descent.
@@ -288,7 +288,7 @@ def adam_optimizer(dataset, net, loss, epochs=1000, rho=(0.9, 0.999), delta=1/1
288288
for j in range(len(weights[i])):
289289
net[i].nodes[j].weights = weights[i][j]
290290

291-
if (e+1) % 10 == 0:
291+
if verbose and (e+1) % verbose == 0:
292292
print("epoch:{}, total_loss:{}".format(e+1,total_loss))
293293
return net
294294

@@ -382,7 +382,7 @@ def get_batch(examples, batch_size=1):
382382
# example of NNs
383383

384384

385-
def neural_net_learner(dataset, hidden_layer_sizes=[4], learning_rate=0.01, epochs=100, optimizer=gradient_descent, batch_size=1):
385+
def neural_net_learner(dataset, hidden_layer_sizes=[4], learning_rate=0.01, epochs=100, optimizer=gradient_descent, batch_size=1, verbose=None):
386386
"""Example of a simple dense multilayer neural network.
387387
:param hidden_layer_sizes: size of hidden layers in the form of a list"""
388388

@@ -399,7 +399,7 @@ def neural_net_learner(dataset, hidden_layer_sizes=[4], learning_rate=0.01, epoc
399399
raw_net.append(DenseLayer(hidden_input_size, output_size))
400400

401401
# update parameters of the network
402-
learned_net = optimizer(dataset, raw_net, mse_loss, epochs, l_rate=learning_rate, batch_size=batch_size)
402+
learned_net = optimizer(dataset, raw_net, mse_loss, epochs, l_rate=learning_rate, batch_size=batch_size, verbose=verbose)
403403

404404
def predict(example):
405405
n_layers = len(learned_net)
@@ -417,7 +417,7 @@ def predict(example):
417417
return predict
418418

419419

420-
def perceptron_learner(dataset, learning_rate=0.01, epochs=100):
420+
def perceptron_learner(dataset, learning_rate=0.01, epochs=100, verbose=None):
421421
"""
422422
Example of a simple perceptron neural network.
423423
"""
@@ -427,7 +427,7 @@ def perceptron_learner(dataset, learning_rate=0.01, epochs=100):
427427
# initialize the network, add dense layer
428428
raw_net = [InputLayer(input_size), DenseLayer(input_size, output_size)]
429429
# update the network
430-
learned_net = gradient_descent(dataset, raw_net, mse_loss, epochs, l_rate=learning_rate)
430+
learned_net = gradient_descent(dataset, raw_net, mse_loss, epochs, l_rate=learning_rate, verbose=verbose)
431431

432432
def predict(example):
433433

‎learning4e.py

Lines changed: 44 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
from utils4e import (
22
removeall, unique, mode, argmax_random_tie, isclose, dotproduct, weighted_sample_with_replacement,
3-
num_or_str, normalize, clip, print_table, open_data, probability, random_weights
3+
num_or_str, normalize, clip, print_table, open_data, probability, random_weights, euclidean_distance
44
)
55

66
import copy
@@ -382,8 +382,8 @@ def cross_validation(learner, size, dataset, k=10, trials=1):
382382
examples = dataset.examples
383383
random.shuffle(dataset.examples)
384384
for fold in range(k):
385-
train_data, val_data = train_test_split(dataset, fold * (n / k),
386-
(fold + 1) * (n / k))
385+
train_data, val_data = train_test_split(dataset, fold * (n // k),
386+
(fold + 1) * (n // k))
387387
dataset.examples = train_data
388388
h = learner(dataset, size)
389389
fold_errs += err_ratio(h, dataset, train_data)
@@ -393,6 +393,37 @@ def cross_validation(learner, size, dataset, k=10, trials=1):
393393
return fold_errs/k
394394

395395

396+
def cross_validation_nosize(learner, dataset, k=10, trials=1):
397+
"""Do k-fold cross_validate and return their mean.
398+
That is, keep out 1/k of the examples for testing on each of k runs.
399+
Shuffle the examples first; if trials>1, average over several shuffles.
400+
Returns Training error, Validataion error"""
401+
k = k or len(dataset.examples)
402+
if trials > 1:
403+
trial_errs = 0
404+
for t in range(trials):
405+
errs = cross_validation(learner, dataset,
406+
k=10, trials=1)
407+
trial_errs += errs
408+
return trial_errs/trials
409+
else:
410+
fold_errs = 0
411+
n = len(dataset.examples)
412+
examples = dataset.examples
413+
random.shuffle(dataset.examples)
414+
for fold in range(k):
415+
train_data, val_data = train_test_split(dataset, fold * (n // k),
416+
(fold + 1) * (n // k))
417+
dataset.examples = train_data
418+
h = learner(dataset)
419+
fold_errs += err_ratio(h, dataset, train_data)
420+
421+
# Reverting back to original once test is completed
422+
dataset.examples = examples
423+
return fold_errs/k
424+
425+
426+
396427
def err_ratio(predict, dataset, examples=None, verbose=0):
397428
"""Return the proportion of the examples that are NOT correctly predicted.
398429
verbose - 0: No output; 1: Output wrong; 2 (or greater): Output correct"""
@@ -521,6 +552,8 @@ def LinearLearner(dataset, learning_rate=0.01, epochs=100):
521552
for example in examples:
522553
x = [1] + example
523554
y = dotproduct(w, x)
555+
# if threshold:
556+
# y = threshold(y)
524557
t = example[idx_t]
525558
err.append(t - y)
526559

@@ -554,17 +587,20 @@ def LogisticLinearLeaner(dataset, learning_rate=0.01, epochs=100):
554587

555588
for epoch in range(epochs):
556589
err = []
590+
h= []
557591
# Pass over all examples
558592
for example in examples:
559593
x = [1] + example
560594
y = 1/(1 + math.exp(-dotproduct(w, x)))
561-
h = [y * (1-y)]
595+
h.append(y * (1-y))
562596
t = example[idx_t]
563597
err.append(t - y)
564598

565599
# update weights
566600
for i in range(len(w)):
567-
w[i] = w[i] + learning_rate * (dotproduct(dotproduct(err,h), X_col[i]) / num_examples)
601+
buffer = [x*y for x,y in zip(err, h)]
602+
# w[i] = w[i] + learning_rate * (dotproduct(err, X_col[i]) / num_examples)
603+
w[i] = w[i] + learning_rate * (dotproduct(buffer, X_col[i]) / num_examples)
568604

569605
def predict(example):
570606
x = [1] + example
@@ -580,6 +616,7 @@ def NearestNeighborLearner(dataset, k=1):
580616
"""k-NearestNeighbor: the k nearest neighbors vote."""
581617
def predict(example):
582618
"""Find the k closest items, and have them vote for the best."""
619+
example.pop(dataset.target)
583620
best = heapq.nsmallest(k, ((dataset.distance(e, example), e)
584621
for e in dataset.examples))
585622
return mode(e[dataset.target] for (d, e) in best)
@@ -829,6 +866,6 @@ def compare(algorithms=None, datasets=None, k=10, trials=1):
829866
Majority(7, 100), Parity(7, 100), Xor(100)] # of datasets
830867

831868
print_table([[a.__name__.replace('Learner', '')] +
832-
[cross_validation(a, d, k, trials) for d in datasets]
869+
[cross_validation_nosize(a, d, k, trials) for d in datasets]
833870
for a in algorithms],
834-
header=[''] + [d.name[0:7] for d in datasets], numfmt='%.2f')
871+
header=[''] + [d.name[0:7] for d in datasets], numfmt='{0:.2f}')

0 commit comments

Comments
 (0)