|
| 1 | +""" |
| 2 | +Know more, visit 莫烦Python: https://morvanzhou.github.io/tutorials/ |
| 3 | +My Youtube Channel: https://www.youtube.com/user/MorvanZhou |
| 4 | +
|
| 5 | +Dependencies: |
| 6 | +torch: 0.1.11 |
| 7 | +matplotlib |
| 8 | +numpy |
| 9 | +""" |
| 10 | +import torch |
| 11 | +from torch.autograd import Variable |
| 12 | +from torch import nn |
| 13 | +from torch.nn import init |
| 14 | +import torch.utils.data as Data |
| 15 | +import torch.nn.functional as F |
| 16 | +import matplotlib.pyplot as plt |
| 17 | +import numpy as np |
| 18 | + |
| 19 | +torch.manual_seed(1) # reproducible |
| 20 | +np.random.seed(1) |
| 21 | + |
| 22 | +# Hyper parameters |
| 23 | +N_SAMPLES = 2000 |
| 24 | +BATCH_SIZE = 64 |
| 25 | +EPOCH = 12 |
| 26 | +LR = 0.03 |
| 27 | +N_HIDDEN = 8 |
| 28 | +ACTIVATION = F.tanh |
| 29 | +B_INIT = -0.2 # use a bad bias constant initializer |
| 30 | + |
| 31 | +# training data |
| 32 | +x = np.linspace(-7, 10, N_SAMPLES)[:, np.newaxis] |
| 33 | +noise = np.random.normal(0, 2, x.shape) |
| 34 | +y = np.square(x) - 5 + noise |
| 35 | + |
| 36 | +# test data |
| 37 | +test_x = np.linspace(-7, 10, 200)[:, np.newaxis] |
| 38 | +noise = np.random.normal(0, 2, test_x.shape) |
| 39 | +test_y = np.square(test_x) - 5 + noise |
| 40 | + |
| 41 | +train_x, train_y = torch.from_numpy(x).float(), torch.from_numpy(y).float() |
| 42 | +test_x = Variable(torch.from_numpy(test_x).float(), volatile=True) # not for computing gradients |
| 43 | +test_y = Variable(torch.from_numpy(test_y).float(), volatile=True) |
| 44 | + |
| 45 | +train_dataset = Data.TensorDataset(data_tensor=train_x, target_tensor=train_y) |
| 46 | +train_loader = Data.DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=2,) |
| 47 | + |
| 48 | +# show data |
| 49 | +plt.scatter(train_x.numpy(), train_y.numpy(), c='#FF9359', s=50, alpha=0.2, label='train') |
| 50 | +plt.legend(loc='upper left') |
| 51 | +plt.show() |
| 52 | + |
| 53 | +class Net(nn.Module): |
| 54 | + def __init__(self, batch_normalization=False): |
| 55 | + super(Net, self).__init__() |
| 56 | + self.do_bn = batch_normalization |
| 57 | + self.fcs = [] |
| 58 | + self.bns = [] |
| 59 | + self.bn_input = nn.BatchNorm1d(1, momentum=0.5) # for input data |
| 60 | + |
| 61 | + for i in range(N_HIDDEN): # build hidden layers and BN layers |
| 62 | + input_size = 1 if i == 0 else 10 |
| 63 | + fc = nn.Linear(input_size, 10) |
| 64 | + setattr(self, 'fc%i' % i, fc) # IMPORTANT set layer to the Module |
| 65 | + self._set_init(fc) # parameters initialization |
| 66 | + self.fcs.append(fc) |
| 67 | + if self.do_bn: |
| 68 | + bn = nn.BatchNorm1d(10, momentum=0.5) |
| 69 | + setattr(self, 'bn%i' % i, bn) # IMPORTANT set layer to the Module |
| 70 | + self.bns.append(bn) |
| 71 | + |
| 72 | + self.predict = nn.Linear(10, 1) # output layer |
| 73 | + self._set_init(self.predict) # parameters initialization |
| 74 | + |
| 75 | + def _set_init(self, layer): |
| 76 | + init.normal(layer.weight, mean=0., std=.1) |
| 77 | + init.constant(layer.bias, B_INIT) |
| 78 | + |
| 79 | + def forward(self, x): |
| 80 | + pre_activation = [x] |
| 81 | + if self.do_bn: x = self.bn_input(x) # input batch normalization |
| 82 | + layer_input = [x] |
| 83 | + for i in range(N_HIDDEN): |
| 84 | + x = self.fcs[i](x) |
| 85 | + pre_activation.append(x) |
| 86 | + if self.do_bn: x = self.bns[i](x) # batch normalization |
| 87 | + x = ACTIVATION(x) |
| 88 | + layer_input.append(x) |
| 89 | + out = self.predict(x) |
| 90 | + return out, layer_input, pre_activation |
| 91 | + |
| 92 | +nets = [Net(batch_normalization=False), Net(batch_normalization=True)] |
| 93 | + |
| 94 | +print(*nets) # print net architecture |
| 95 | + |
| 96 | +opts = [torch.optim.Adam(net.parameters(), lr=LR) for net in nets] |
| 97 | + |
| 98 | +loss_func = torch.nn.MSELoss() |
| 99 | + |
| 100 | +f, axs = plt.subplots(4, N_HIDDEN+1, figsize=(10, 5)) |
| 101 | +plt.ion() # something about plotting |
| 102 | +plt.show() |
| 103 | + |
| 104 | +def plot_histogram(l_in, l_in_bn, pre_ac, pre_ac_bn): |
| 105 | + for i, (ax_pa, ax_pa_bn, ax, ax_bn) in enumerate(zip(axs[0, :], axs[1, :], axs[2, :], axs[3, :])): |
| 106 | + [a.clear() for a in [ax_pa, ax_pa_bn, ax, ax_bn]] |
| 107 | + if i == 0: |
| 108 | + p_range = (-7, 10) |
| 109 | + the_range = (-7, 10) |
| 110 | + else: |
| 111 | + p_range = (-4, 4) |
| 112 | + the_range = (-1, 1) |
| 113 | + ax_pa.set_title('L' + str(i)) |
| 114 | + ax_pa.hist(pre_ac[i].data.numpy().ravel(), bins=10, range=p_range, color='#FF9359', alpha=0.5) |
| 115 | + ax_pa_bn.hist(pre_ac_bn[i].data.numpy().ravel(), bins=10, range=p_range, color='#74BCFF', alpha=0.5) |
| 116 | + ax.hist(l_in[i].data.numpy().ravel(), bins=10, range=the_range, color='#FF9359') |
| 117 | + ax_bn.hist(l_in_bn[i].data.numpy().ravel(), bins=10, range=the_range, color='#74BCFF') |
| 118 | + for a in [ax_pa, ax, ax_pa_bn, ax_bn]: |
| 119 | + a.set_yticks(()) |
| 120 | + a.set_xticks(()) |
| 121 | + ax_pa_bn.set_xticks(p_range) |
| 122 | + ax_bn.set_xticks(the_range) |
| 123 | + axs[0, 0].set_ylabel('PreAct') |
| 124 | + axs[1, 0].set_ylabel('BN PreAct') |
| 125 | + axs[2, 0].set_ylabel('Act') |
| 126 | + axs[3, 0].set_ylabel('BN Act') |
| 127 | + plt.pause(0.01) |
| 128 | + |
| 129 | +# training |
| 130 | +losses = [[], []] # recode loss for two networks |
| 131 | +for epoch in range(EPOCH): |
| 132 | + print('Epoch: ', epoch) |
| 133 | + layer_inputs, pre_acts = [], [] |
| 134 | + for net, l in zip(nets, losses): |
| 135 | + net.eval() # set eval mode to fix moving_mean and moving_var |
| 136 | + pred, layer_input, pre_act = net(test_x) |
| 137 | + l.append(loss_func(pred, test_y).data[0]) |
| 138 | + layer_inputs.append(layer_input) |
| 139 | + pre_acts.append(pre_act) |
| 140 | + net.train() # free moving_mean and moving_var |
| 141 | + plot_histogram(*layer_inputs, *pre_acts) # plot histogram |
| 142 | + |
| 143 | + for step, (b_x, b_y) in enumerate(train_loader): |
| 144 | + b_x, b_y = Variable(b_x), Variable(b_y) |
| 145 | + for net, opt in zip(nets, opts): # train for each network |
| 146 | + pred, _, _ = net(b_x) |
| 147 | + loss = loss_func(pred, b_y) |
| 148 | + opt.zero_grad() |
| 149 | + loss.backward() |
| 150 | + opt.step() # it will also learn the parameters in Batch Normalization |
| 151 | + |
| 152 | + |
| 153 | +plt.ioff() |
| 154 | + |
| 155 | +# plot training loss |
| 156 | +plt.figure(2) |
| 157 | +plt.plot(losses[0], c='#FF9359', lw=3, label='Original') |
| 158 | +plt.plot(losses[1], c='#74BCFF', lw=3, label='Batch Normalization') |
| 159 | +plt.xlabel('step') |
| 160 | +plt.ylabel('test loss') |
| 161 | +plt.ylim((0, 2000)) |
| 162 | +plt.legend(loc='best') |
| 163 | + |
| 164 | +# evaluation |
| 165 | +# set net to eval mode to freeze the parameters in batch normalization layers |
| 166 | +[net.eval() for net in nets] # set eval mode to fix moving_mean and moving_var |
| 167 | +preds = [net(test_x)[0] for net in nets] |
| 168 | +plt.figure(3) |
| 169 | +plt.plot(test_x.data.numpy(), preds[0].data.numpy(), c='#FF9359', lw=4, label='Original') |
| 170 | +plt.plot(test_x.data.numpy(), preds[1].data.numpy(), c='#74BCFF', lw=4, label='Batch Normalization') |
| 171 | +plt.scatter(test_x.data.numpy(), test_y.data.numpy(), c='r', s=50, alpha=0.2, label='train') |
| 172 | +plt.legend(loc='best') |
| 173 | +plt.show() |
0 commit comments