Backpropogation

import numpy as np
import pandas as pd
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt

# Data preparation
data = load_iris()
X_train, X_test, y_train, y_test = train_test_split(data.data, data.target, test_size=20, random_state=4)

# Hyperparameters
learning_rate, iterations = 0.1, 5000
input_size, hidden_size, output_size = 4, 2, 3

# Initialization
W1, W2 = np.random.normal(scale=0.5, size=(input_size, hidden_size)), np.random.normal(scale=0.5, size=(hidden_size, output_size))

# Activation function
sigmoid = lambda x: 1 / (1 + np.exp(-x))

# Error and accuracy functions
mse = lambda y_pred, y_true: ((y_pred - np.eye(output_size)[y_true].reshape(y_pred.shape))**2).sum() / (2*y_pred.size)
accuracy = lambda y_pred, y_true: (y_pred.argmax(axis=1) == y_true.argmax(axis=1)).mean()

# Training loop
results = pd.DataFrame(columns=["mse", "accuracy"])
for _ in range(iterations):
    A1, A2 = sigmoid(np.dot(X_train, W1)), sigmoid(np.dot(sigmoid(np.dot(X_train, W1)), W2))
    results = pd.concat([results, pd.DataFrame({"mse": [mse(A2, y_train)], "accuracy": [accuracy(A2, y_train)]})], ignore_index=True)
    
    # Backpropagation
    E1, dW1 = A2 - np.eye(output_size)[y_train], E1 * A2 * (1 - A2)
    E2, dW2 = np.dot(dW1, W2.T), E2 * A1 * (1 - A1)
    W2, W1 = W2 - learning_rate * np.dot(A1.T, dW1) / y_train.size, W1 - learning_rate * np.dot(X_train.T, dW2) / y_train.size

# Plotting
results.mse.plot(title="Mean Squared Error")
plt.show()
results.accuracy.plot(title="Accuracy")
plt.show()

# Testing
A2_test = sigmoid(np.dot(sigmoid(np.dot(X_test, W1)), W2))
test_acc = accuracy(A2_test, y_test)
print("Test accuracy: {}".format(test_acc))