# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# ## 0. What We Are Going To Do:
# We are going to classify images of handwritten digits (MNIST dataset) using a fully-connected neural network.
#
# After successful training, our model will be able to guess the digit in the image.
#
# IMPORTANT NOTE: Some part of the code is already completed. Do not delete them!

# ** In this section we import all the packages you need(You may edit & rerun this part later)**
#
# Note: You can also import classes/functions within a package directly (e.g. from keras.layers import Dense).
#

import keras
import matplotlib.pyplot as plt #This package is for plotting
# %matplotlib inline  
import numpy as np
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense, Input
from keras.optimizers import SGD
from keras.initializers import RandomNormal
from keras.models import load_model

import pandas as pd
import random

# ## 1. Prepare Data:
#
# The dataset is loaded in this section. 
#
# **1-0. Check the dimensions of data and its minimum & maximum.**
#
# Note: You should paste the dataset file in '~/.keras/datasets/' directory or it will start downloading it.

(x_train, y_train), (x_test, y_test) = mnist.load_data()

# +
print('train data dim:', x_train.shape)
# Print x_test and y_test dimensions
print('test data dim:', x_test.shape)

# Question: How many train/test samples does it have?
print('train examples:',len(x_train))
print('test examples:',len(x_test))
# Print the minimum and maximum of x_train and x_test(use numpy min and max functions)
print('x_train => max : {} , min : {}'.format(np.max(x_train),np.min(x_train)))
print('x_test => max : {} , min : {}'.format(np.max(x_test),np.min(x_test)))
# -

# ** 1-1. Check a random sample of train data and its label. **

# generate a random number. (use numpy random.randint)
rand_num = np.random.randint(1,59999)
# plot using plt.imshow() & plt.show()
plt.imshow(x_train[rand_num],cmap='gray')
plt.show()
# print its label
print(y_train[rand_num])

# **1-2. Our Network accept 1D data. So we should flatten our 2D image, then print the dimension of the result arrays.**
#
# Use numpy reshape function.
#

# Reshape x_train and x_test using numpy reshape function
x_train = x_train.reshape(-1,28*28)
x_test = x_test.reshape(-1,28*28)

# ** 1-3. Normalize data by rescaling them to (0,1) **

# normalize x_train and x_test by dividing them by max of x_train:
x_train = x_train/np.max(x_train)
x_test = x_test/np.max(x_train)
np.max(x_train)

# ** 1-4. Convert label arrays to 1-hot representation **

# Use keras.utils.to_categorical
y_train= keras.utils.to_categorical(y_train, num_classes=10)
y_train

y_test= keras.utils.to_categorical(y_test, num_classes=10)
y_test

# ## 2. Define Model

# ** 2-0. Add the following layers to the network:**
# * Hidden Layer 1: Fully Conncted + Relu Activition (e.g. 512 Nuerons)
# * Hidden Layer 2: Fully Connected + Relu Activition (e.g. 512 Neurons)
# * Outout Layer: Fully Connected + Softmax Activition

x_train.shape

model = Sequential()
# Add the layers to model here.
# Hidden Layer1: 512 Neurons+ relu activation + Normal Initialization mean=0, std = 0.1(RandomNormal). You Should define input shape for layer1
model.add(Dense(512, activation='relu', input_shape=(784,), kernel_initializer= RandomNormal(0,0.1)))
# Hidden Layer2: 512 Neurons+ relu activation + Normal Initialization mean=0, std = 0.1(RandomNormal).
model.add(Dense(512, activation='relu', kernel_initializer= RandomNormal(0,0.1)))
# Output Layer1: Question: How many neurons?what kind of activation function? + Normal Initialization mean=0, std = 0.1(RandomNormal).
model.add(Dense(10, activation='softmax', kernel_initializer= RandomNormal(0,0.1)))

# ** 2-1. Determine loss function, optimizer and metrics for the model **

# define the optimizer and its learning rate
sgd = SGD(lr=0.01)
# Use categorical_crossentropy as loss function, and accuracy as one of the metrics (you may add other metrics too). 
# use model.compile to add the above parameters to the model
model.compile(loss='categorical_crossentropy',
              optimizer='sgd',
              metrics=['accuracy'])

# ** 2-2. Print the review of the model **

model.summary()
# Here we saved the raw model without any training. we will use it later.
model.save('raw_model.h5')

# ## 3. Train And Evaluate Model. 

# ** 3-0. Train model on training data using model.fit for 3 epoch**

# Use model.fit to train the model. Following hyperparameters are suggested (but you may change them if you want):
# batch_size = 32
# epochs = 3
# Validation data percentage: 20%
# use verbose for logging the training
model.fit(x = x_train, y= y_train, batch_size=32, epochs=3,
          verbose=1, callbacks=None, validation_split=0.2, validation_data=None, shuffle=True,
          class_weight=None, sample_weight=None, initial_epoch=0, steps_per_epoch=None, validation_steps=None)

# ** 3-1. Evaluate model on test data using model.evaluate. Print the model accuracy on test set. **

model.evaluate(x=x_test, y=y_test, batch_size=None, verbose=1, sample_weight=None, steps=None)

# ** 3-2. Save model **
#
# In Keras, you can save the model to a HDF5 file(.h5) and reload it later simply by model.save(filepath) and keras.models.load_model(filepath), respectively.
#
# The saved model contains:
# * the architecture of the model, allowing to re-create the model
# * the weights of the model
# * the training configuration (loss, optimizer)
# * the state of the optimizer, allowing to resume training exactly where you left off.
#

# save model here:
model.save('mlp.h5')
# Delete model to make sure you reload it correctly:
del model

# ** 3-3. Load model and Predict label for a random image in train set. Verify predicted label by ploting the image. **

# reload the model here:
model = load_model('mlp.h5')
# generate a random number. (use numpy random.randint) and use model.predict to predict its label. 
rand = np.random.randint(1,60000)
img = x_train[rand]
img = np.expand_dims(img,axis=0)
pre_label= model.predict(img, batch_size=None, verbose=0, steps=None)
index = np.argmax(pre_label)
print(y_train[rand])
print(np.argmax(y_train[rand]))
# print its label
print(index)

# ** 3-4. Continue training + Callbacks **
#
# Please Examine how these callbacks work. you may add other callbacks to this list.

# +
# We will use two callbacks here: EarlyStopping, CSVLogger (you may add other callbacks to this list)
callback = [keras.callbacks.EarlyStopping(monitor='val_acc', verbose=1, min_delta=0.01, patience = 2, mode = 'max'),
            keras.callbacks.CSVLogger('log.csv')]
history = model.fit(x_train, y_train,
                    batch_size = 32,
                    epochs = 10,
                    verbose = 1,
                    validation_split = 0.2,
                    callbacks = callback)

plt.figure(figsize=(5,3))
plt.plot(history.epoch,history.history['loss'])
plt.title('loss')

plt.figure(figsize=(5,3))
plt.plot(history.epoch,history.history['acc'])
plt.title('accuracy');
# -

# ## 4. Extras

# ** 4-0. Initialization is important! **
#
# Redefine Network, this time use mean=0 and std=1 for initialization.
#
# Then Try to fit the model.
#
# Repeat this with other initialization methods and parameters.

# +
# bad model!!!
# import new functions
from keras.optimizers import Adagrad
from keras.initializers import TruncatedNormal

model2 = Sequential()
# Add the layers to model here.
model2.add(Dense(1024, activation='relu', input_shape=(784,), kernel_initializer= TruncatedNormal(0,1)))
# Hidden Layer2: 512 Neurons+ relu activation + Normal Initialization mean=0, std = 0.1(RandomNormal).
model2.add(Dense(512, activation='relu', kernel_initializer= TruncatedNormal(0,1)))
# Output Layer1: Question: How many neurons?what kind of activation function? + Normal Initialization mean=0, std = 0.1(RandomNormal).
model2.add(Dense(10, activation='softmax', kernel_initializer= TruncatedNormal(0,1)))

# define the optimizer and its learning rate
adagrad = Adagrad(lr=0.01, epsilon=None, decay=0.0)
# Use categorical_crossentropy as loss function, and accuracy as one of the metrics (you may add other metrics too). 
# use model.compile to add the above parameters to the model
model2.compile(loss='categorical_crossentropy',
              optimizer=adagrad,
              metrics=['accuracy'])

# Use model.fit to train the model. Following hyperparameters are suggested (but you may change them if you want):
# batch_size = 32
# epochs = 3
# Validation data percentage: 20%
model2.fit(x = x_train, y= y_train, batch_size=32, epochs=3,
          verbose=0, callbacks=None, validation_split=0.2, validation_data=None, shuffle=True,
          class_weight=None, sample_weight=None, initial_epoch=0, steps_per_epoch=None, validation_steps=None)

model2.evaluate(x=x_test, y=y_test, batch_size=None, verbose=1, sample_weight=None, steps=None)
# -

# ** 4-1. Overfitting/Underfitting **
#
# Load the 'raw_model.h5' and this time use 1 percent of training data for training, and all test data for validation.
#

del model
# Load model here and fit it with less data
model = load_model('raw_model.h5')
model.fit(x = x_train[0:int((len(x_train)/100)),], y= y_train[0:int((len(x_train)/100)),], batch_size=32, epochs=3, validation_split=0.2)
model.evaluate(x=x_test, y=y_test, batch_size=None, verbose=0, sample_weight=None, steps=None)

# ** We swee the model is underfitted!**

# Now create another fully-connected model with 1 hidden layer(5 Neurons). Use all training data for training this model. (10 Training epoch)
#
# Observe the result.

# +
model3 = Sequential()
model3.add(Dense(5, activation='relu', input_shape=(784,), kernel_initializer= RandomNormal(0,1)))
model3.add(Dense(10, activation='softmax', kernel_initializer= RandomNormal(0,1)))

# define the optimizer and its learning rate
sgd = SGD(lr=0.01)
# Use categorical_crossentropy as loss function, and accuracy as one of the metrics (you may add other metrics too). 
# use model.compile to add the above parameters to the model
model3.compile(loss='categorical_crossentropy',
              optimizer=sgd,
              metrics=['accuracy'])

# Use model.fit to train the model. Following hyperparameters are suggested (but you may change them if you want):
# batch_size = 32
# epochs = 3
# Validation data percentage: 20%
model3.fit(x = x_train, y= y_train, batch_size=32, epochs=3,
        validation_split=0.2, shuffle=True)

model3.evaluate(x=x_test, y=y_test, batch_size=None, verbose=0, sample_weight=None, steps=None)
# -

# **Not enough hidden layers and neurons!**



StarcoderdataJupyterScriptsDedupFiltered