Before you turn this problem in, make sure everything runs as expected.
First, restart the kernel
(in the menubar, select Kernel→Restart) and then run all cells (in the menubar, select Cell→Run
All).
Make sure you fill in any place that says YOUR CODE HERE or "YOUR ANSWER HERE", as well
as your name and collaborators below:
You may remove raise NotImplementedError() line and write your own code.
NAME = ""
WSU_ID = ""
Assignment 1
Q1. Task Description
1. Data Generation:
• Generate two normal distributions each of size 30,000 with feature dimension=100.
(Hint: Shape(2*30000, 100))
• You may choose arbitrary mean and standard deviation.
• Label the first set of feature vectors as "+1" and the second set of feature vectors as
either "-1" or "0."
alt text
import numpy as np
# Step 1: Generate Data
np.random.seed(42) # For reproducibility
# Generate two normal distributions
mean1 = np.random.randn(1, 100)
mean2 = np.random.randn(1, 100)
std_dev = 1.0
data1 = mean1 + std_dev * np.random.randn(30000, 100)
data2 = mean2 + std_dev * np.random.randn(30000, 100)
labels1 = np.ones(30000) # Label the first set as "+1" or "-1"
labels2 = np.zeros(30000)
# Combine data and labels
data = np.vstack((data1, data2))
labels = np.concatenate((labels1, labels2), axis=0)
2. Data Splitting
• Split the generated data into training, validation and test sets in 8:1:1 ratio.
• You are expected to write the code from scratch. (i.e. Do not use train_test_split function
from sklearn)
# Split the data into training (80%), validation (10%), and test (10%)
sets manually
total_samples = len(data)
split_indices = [0.8, 0.9] # Split ratios
split_sizes = [int(split * total_samples) for split in split_indices]
x_train, x_val, x_test = np.split(data, split_sizes, axis=0)
y_train, y_val, y_test = np.split(labels, split_sizes, axis=0)
x_train.shape
(48000, 100)
3. Classifier Implementation
• Implement Linear Classifiers using three different methods. (i.e. Write code from scratch.
Refer lecture slides)
– Random Method
– Perceptron Method
– Gradient Descent
import numpy as np
# Step 3: Classifier Implementation - Random Method with Epochs
def random_classifier(x, y_train, X_val, y_val, num_epochs=10):
"""
Randomly assigns labels (+1 or -1) to input data x for a specified
number of epochs.
Args:
x (numpy.ndarray): Input data with shape (num_samples,
num_features).
num_epochs (int): Number of training epochs.
Returns:
predictions (numpy.ndarray): Randomly assigned labels (+1 or -1)
with shape (num_samples,).
val_accuracy (list): List of validation accuracies after each
epoch.
"""
num_samples = x.shape[0]
val_accuracy = []
for epoch in range(num_epochs):
# Assign random labels for this epoch
predictions = np.random.choice([0, 1], size=num_samples)
# Calculate and print training accuracy for this epoch
correct = np.sum(y_train == predictions)
accuracy = correct / len(y_train)
print(f"Epoch {epoch + 1}/{num_epochs} - Training Accuracy:
{accuracy:.2f}")
predictions = np.random.choice([0, 1], size=len(X_val))
correct = np.sum(y_val == predictions)
val_accuracy.append(correct / len(y_val))
return predictions, val_accuracy
num_epochs = 10
# Use the random classifier with epochs
random_predictions, val_accuracy = random_classifier(x_train, y_train,
x_val, y_val, num_epochs=num_epochs)
# Print validation accuracies after each epoch
for epoch, val_acc in enumerate(val_accuracy):
print(f"Epoch {epoch + 1}/{num_epochs} - Validation Accuracy:
{val_acc:.2f}")
Epoch 1/10 - Training Accuracy: 0.50
Epoch 2/10 - Training Accuracy: 0.50
Epoch 3/10 - Training Accuracy: 0.50
Epoch 4/10 - Training Accuracy: 0.50
Epoch 5/10 - Training Accuracy: 0.50
Epoch 6/10 - Training Accuracy: 0.50
Epoch 7/10 - Training Accuracy: 0.50
Epoch 8/10 - Training Accuracy: 0.50
Epoch 9/10 - Training Accuracy: 0.50
Epoch 10/10 - Training Accuracy: 0.50
Epoch 1/10 - Validation Accuracy: 0.50
Epoch 2/10 - Validation Accuracy: 0.50
Epoch 3/10 - Validation Accuracy: 0.50
Epoch 4/10 - Validation Accuracy: 0.50
Epoch 5/10 - Validation Accuracy: 0.50
Epoch 6/10 - Validation Accuracy: 0.50
Epoch 7/10 - Validation Accuracy: 0.50
Epoch 8/10 - Validation Accuracy: 0.50
Epoch 9/10 - Validation Accuracy: 0.50
Epoch 10/10 - Validation Accuracy: 0.51
import numpy as np
class Perceptron:
def __init__(self, learning_rate, epochs):
self.weights = None
self.bias = None
self.learning_rate = learning_rate
self.epochs = epochs
def activation(self, z):
return np.heaviside(z, 0)
def fit(self, X, y, X_val, y_val):
n_features = X.shape[1]
# Initializing weights and bias
self.weights = np.zeros((n_features))
self.bias = 0
train_accuracies = []
val_accuracies = []
# Iterating until the number of epochs
for epoch in range(self.epochs):
# Traversing through the entire training set
for i in range(len(X)):
z = np.dot(X, self.weights) + self.bias
y_pred = self.activation(z)
# Updating weights and bias
self.weights = self.weights + self.learning_rate *
(y[i] - y_pred[i]) * X[i]
self.bias = self.bias + self.learning_rate * (y[i] -
y_pred[i])
# Calculate training accuracy for this epoch
train_accuracy = np.mean(self.predict(X) == y)
train_accuracies.append(train_accuracy)
# Calculate validation accuracy for this epoch
val_accuracy = np.mean(self.predict(X_val) == y_val)
val_accuracies.append(val_accuracy)
print(f"Epoch {epoch + 1}/{self.epochs} - Training
Accuracy: {train_accuracy:.2f} - Validation Accuracy:
{val_accuracy:.2f}")
return self.weights, self.bias, train_accuracies,
val_accuracies
def predict(self, X):
z = np.dot(X, self.weights) + self.bias
return self.activation(z)
perceptron = Perceptron(0.01, 3)
# Train the model with your training data and validate on validation
data
trained_weights, trained_bias, train_accuracies, val_accuracies =
perceptron.fit(x_train, y_train, x_val, y_val)
# If you have a separate test dataset (replace `test_data` with your
test data)
test_predictions = perceptron.predict(x_test)
# Print the trained weights and bias
print("Trained Weights:", trained_weights)
print("Trained Bias:", trained_bias)
# Print the predictions on the test data
print("Test Predictions:", test_predictions)
# Print the validation accuracies over epochs
print("Validation Accuracies:", val_accuracies)
Epoch 1/3 - Training Accuracy: 1.00 - Validation Accuracy: 1.00
Epoch 2/3 - Training Accuracy: 1.00 - Validation Accuracy: 1.00
Epoch 3/3 - Training Accuracy: 1.00 - Validation Accuracy: 1.00
Trained Weights: [ 0.02930894 0.01577013 0.02865582 0.0143475
0.00191648 -0.01829453
-0.00588742 0.01397218 0.00327979 0.05439526 0.01756178 -
0.00475071
0.01187068 -0.04555749 -0.01602566 0.00112243 -0.01589925 0.013728
-0.01856712 -0.01436198 0.02043848 -0.00858916 -0.01994088 -
0.0220679
-0.02221556 -0.00416195 -0.02028789 0.00435762 -0.00615626
0.01356935
0.01116244 0.01624046 0.01467706 -0.00285822 0.03518707 -
0.01609795
-0.01171295 -0.00303621 -0.01721151 0.02555033 -0.00350199 -
0.02203814
0.0052322 0.00163387 -0.04067547 -0.02737306 0.00533138
0.01603403
0.00976284 -0.02873596 0.00402219 -0.0099823 0.02706815
0.00378555
-0.01541295 0.00844676 -0.00940219 -0.0171186 0.02834484
0.01975563
0.00611272 0.0021402 -0.04979813 -0.01340293 -0.00384768
0.00251796
0.00144038 -0.01788196 0.00655632 -0.00263312 0.01251551
0.01948608
0.01936723 0.01471368 -0.04047635 -0.00654433 0.00207036 -
0.01988373
0.0082425 -0.04606465 -0.00845419 0.02105961 0.03866535 -
0.01093569
0.02134901 -0.03409519 0.01080157 0.01874172 0.00422245 -
0.00257807
-0.00859017 -0.00083997 -0.02720405 0.03081453 -0.00510074 -0.033766
0.02433598 0.00608974 0.01748785 0.00074318]
Trained Bias: 0.0
Test Predictions: [0. 0. 0. ... 0. 0. 0.]
Validation Accuracies: [1.0, 1.0, 1.0]
alpha = 0.01 # Step size
iterations = 10 # No. of iterations
m = y_train.size # No. of training data points
np.random.seed(123) # Set the seed
theta = np.random.rand(x_train.shape[1]) # Pick some random values to
start with, including the bias term
# GRADIENT DESCENT
def gradient_descent(X, y, theta, iterations, alpha):
past_costs = []
past_thetas = [theta]
for i in range(iterations):
prediction = np.dot(X, theta)
error = prediction - y
cost = 1 / (2 * m) * np.dot(error.T, error)
past_costs.append(cost)
theta = theta - (alpha * (1 / m) * np.dot(X.T, error))
past_thetas.append(theta)
return past_thetas, past_costs
# Pass the relevant variables to the function and get the new values
back...
past_thetas, past_costs = gradient_descent(x_train, y_train, theta,
iterations, alpha)
theta = past_thetas[-1]
# Print the results...
print("Gradient Descent Weights:", theta[1:]) # Excluding the bias
term
print("Gradient Descent Bias:", theta[0]) # Bias term
# Training Accuracy Calculation
def calculate_accuracy(X, y, theta):
prediction = np.dot(X, theta)
predicted_labels = np.where(prediction >= 0, 1, 0)
accuracy = np.mean(predicted_labels == y)
return accuracy
training_accuracy = calculate_accuracy(x_train, y_train, theta)
validation_accuracy = calculate_accuracy(x_val, y_val, theta)
print("Training Accuracy:", training_accuracy)
print("Validation Accuracy:", validation_accuracy)
Gradient Descent Weights: [ 0.25627177 0.25038837 0.59917103
0.63970942 0.36930991 0.97167784
0.66674311 0.40117186 0.39193994 0.29455745 0.63114886
0.41072971
-0.08245583 0.25614801 0.6345314 0.10644166 0.18772993
0.41568106
0.39119027 0.65677765 0.75926664 0.64870266 0.47832048
0.61519105
0.28286365 0.26157788 0.23032212 0.23021395 0.55619403
0.05641231
0.50334782 0.39370568 0.37749935 0.44011723 0.19412849
0.40671639
0.6847459 0.76826794 0.47578244 0.60560003 0.10481406
0.28965287
0.3563486 0.68623067 0.17912887 0.42248432 0.96412374
0.48772846
0.4430716 0.12748256 0.72307007 0.50990833 0.531048
0.37290001
0.33818183 0.31291364 0.59443465 0.81903494 0.52219904
0.58004777
0.51087395 0.49544815 0.54350474 0.80537081 0.1565376
0.67935071
0.26833345 0.1985203 0.48198217 0.11375451 0.90546796
0.5662372
0.75083446 -0.15106624 0.58440185 0.50743967 0.11817427
0.14637337
0.48298345 0.26844331 0.65397454 0.60194814 0.31766682
0.78773821
0.72206429 0.37451293 0.05501134 0.25415076 0.40143725
0.64862934
0.9518089 0.27900408 0.68120126 0.51204179 0.53185254
0.16313734
0.37971075 0.21964968 0.30610183]
Gradient Descent Bias: 0.6719870624353718
Training Accuracy: 0.5470833333333334
Validation Accuracy: 0.5073333333333333
4. Performance Evaluation
• Evaluate the performance of each method on the test set using various evaluation
metrics discussed in the class, such as accuracy, precision, recall, F1-score. (You are
expected to write the code from scratch for each of the metrics.)
import numpy as np
# Define evaluation functions
def accuracy(y_true, y_pred):
"""
Calculate accuracy.
Args:
y_true (numpy.ndarray): True labels with shape (num_samples,).
y_pred (numpy.ndarray): Predicted labels with shape
(num_samples,).
Returns:
float: Accuracy.
"""
correct = np.sum(y_true == y_pred)
total = len(y_true)
return correct / total
def precision(y_true, y_pred):
"""
Calculate precision.
Args:
y_true (numpy.ndarray): True labels with shape (num_samples,).
y_pred (numpy.ndarray): Predicted labels with shape
(num_samples,).
Returns:
float: Precision.
"""
true_positive = 0
false_positive = 0
for i in range(len(y_true)):
if y_true[i] == 0 and y_pred[i] == 0:
true_positive += 1
elif y_true[i] == 0 and y_pred[i] == 1:
false_positive += 1
# Check for zero denominator
if true_positive + false_positive == 0:
return 0.0 # Return 0 precision if denominator is zero
return true_positive / (true_positive + false_positive)
def recall(y_true, y_pred):
"""
Calculate recall.
Args:
y_true (numpy.ndarray): True labels with shape (num_samples,).
y_pred (numpy.ndarray): Predicted labels with shape
(num_samples,).
Returns:
float: Recall.
"""
true_positive = 0
false_negative = 0
for i in range(len(y_true)):
if y_true[i] == 0 and y_pred[i] == 0:
true_positive += 1
elif y_true[i] == 0 and y_pred[i] == 1:
false_negative += 1
return true_positive / (true_positive + false_negative)
def f1_score(y_true, y_pred):
"""
Calculate F1-score.
Args:
y_true (numpy.ndarray): True labels with shape (num_samples,).
y_pred (numpy.ndarray): Predicted labels with shape
(num_samples,).
Returns:
float: F1-score.
"""
prec = precision(y_true, y_pred)
rec = recall(y_true, y_pred)
# Check for zero denominator in precision or recall
if prec == 0.0 or rec == 0.0:
return 0.0 # Return 0 F1-score if precision or recall is zero
return 2 * (prec * rec) / (prec + rec)
# Ex
# Predict using the trained model (w and bias obtained from training)
random_pred = np.random.choice([0, 1], size=len(x_test))
# Calculate evaluation metrics
acc = accuracy(y_test, random_pred)
prec = precision(y_test, random_pred)
rec = recall(y_test, random_pred)
f1 = f1_score(y_test, random_pred)
# Print the results
print(f"Random Accuracy: {acc:.2f}")
print(f"Random Precision: {prec:.2f}")
print(f"Random Recall: {rec:.2f}")
print(f"Random F1-Score: {f1:.2f}")
prediction = np.dot(x_test, theta)
grad_predicted = np.where(prediction >= 0, 1, 0)
# Calculate evaluation metrics
acc = accuracy(y_test, grad_predicted)
prec = precision(y_test, grad_predicted)
rec = recall(y_test, grad_predicted)
f1 = f1_score(y_test, grad_predicted)
# Print the results
print(f"Grad Accuracy: {acc:.2f}")
print(f"Grad Precision: {prec:.2f}")
print(f"Grad Recall: {rec:.2f}")
print(f"Grad F1-Score: {f1:.2f}")
preceptron_pred = perceptron.predict(x_test)
acc = accuracy(y_test, preceptron_pred)
prec = precision(y_test, preceptron_pred)
rec = recall(y_test, preceptron_pred)
f1 = f1_score(y_test, preceptron_pred)
# Print the results
print(f"Perceptron Accuracy: {acc:.2f}")
print(f"Preceptron Precision: {prec:.2f}")
print(f"Preceptron Recall: {rec:.2f}")
print(f"Perceptron F1-Score: {f1:.2f}")
Random Accuracy: 0.50
Random Precision: 0.50
Random Recall: 0.50
Random F1-Score: 0.50
Grad Accuracy: 0.51
Grad Precision: 0.51
Grad Recall: 0.51
Grad F1-Score: 0.51
Perceptron Accuracy: 1.00
Preceptron Precision: 1.00
Preceptron Recall: 1.00
Perceptron F1-Score: 1.00
5. Determine the best method
• Analyze the evaluation results to determine which method is the best fit for solving this
problem based on the chosen evaluation metrics. (Write your reasoning)
Best Method is Perceptron and it was decided based on the accuracy, precision, recall, F1-
Score
By following these steps, you can systematically generate, split, implement, and
evaluate the performance of different linear classification methods to identify the
most suitable one for your specific problem.
Extra Credits
• Implement cross validation on the best method and show the results. (Refer the lecture
slides for more information)
import numpy as np
class Perceptron:
def __init__(self, learning_rate, epochs):
self.weights = None
self.bias = None
self.learning_rate = learning_rate
self.epochs = epochs
def activation(self, z):
return np.heaviside(z, 0)
def fit(self, X, y):
n_features = X.shape[1]
# Initializing weights and bias
self.weights = np.zeros((n_features))
self.bias = 0
# Iterating until the number of epochs
for epoch in range(self.epochs):
# Traversing through the entire training set
for i in range(len(X)):
z = np.dot(X, self.weights) + self.bias
y_pred = self.activation(z)
# Updating weights and bias
self.weights = self.weights + self.learning_rate *
(y[i] - y_pred[i]) * X[i]
self.bias = self.bias + self.learning_rate * (y[i] -
y_pred[i])
def predict(self, X):
z = np.dot(X, self.weights) + self.bias
return self.activation(z)
def k_fold_cross_validation(X, y, k, learning_rate, epochs):
fold_size = len(X) // k
val_accuracies = []
for i in range(k):
# Split the data into training and validation sets for this
fold
val_start = i * fold_size
val_end = (i + 1) * fold_size
x_val_fold = X[val_start:val_end]
y_val_fold = y[val_start:val_end]
x_train_fold = np.concatenate((X[:val_start], X[val_end:]),
axis=0)
y_train_fold = np.concatenate((y[:val_start], y[val_end:]),
axis=0)
# Create and train the Perceptron model
perceptron = Perceptron(learning_rate, epochs)
perceptron.fit(x_train_fold, y_train_fold)
# Calculate validation accuracy for this fold
val_accuracy = np.mean(perceptron.predict(x_val_fold) ==
y_val_fold)
val_accuracies.append(val_accuracy)
return val_accuracies
learning_rate = 0.01
epochs = 3
k = 5 # Number of folds for cross-validation
# Perform k-fold cross-validation
val_accuracies = k_fold_cross_validation(x_train, y_train, k,
learning_rate, epochs)
# Print the validation accuracies for each fold
for i, val_accuracy in enumerate(val_accuracies):
print(f"Fold {i + 1} Validation Accuracy: {val_accuracy:.2f}")
# Calculate the average validation accuracy over all folds
average_val_accuracy = np.mean(val_accuracies)
print(f"Average Validation Accuracy: {average_val_accuracy:.2f}")
