# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] id="s2YwmCWsKiyK" colab_type="text"
# COPYRIGHT © 2018 <NAME> <<EMAIL>>

# + [markdown] id="wBwx7thMVEJU" colab_type="text"
# ### Setup

# + id="2RaDNJFYkOzA" colab_type="code" colab={"autoexec": {"startup": false, "wait_interval": 0}, "base_uri": "https://localhost:8080/", "height": 163} outputId="b90ac59b-741c-43c1-b10d-e133424b297f" executionInfo={"status": "ok", "timestamp": 1530976675837, "user_tz": -60, "elapsed": 8187, "user": {"displayName": "<NAME>", "photoUrl": "//lh4.googleusercontent.com/-8IeZZ41ybbo/AAAAAAAAAAI/AAAAAAAACkI/_LkIlaW8I6g/s50-c-k-no/photo.jpg", "userId": "105201401933439284277"}}
# install dependencies
# !rm -r Neural_Networks-101-demo
# !git clone -b explanations https://github.com/KiranArun/Neural_Networks-101-demo.git
# !python3 /content/Neural_Networks-101-demo/scripts/setup.py helper_funcs

# + [markdown] id="4384DJ_skxW3" colab_type="text"
# # MNIST Handwritten Digits Classifier

# + [markdown] id="CRFq1dEwT8HK" colab_type="text"
# ## Overview

# + id="oQAM2f_gkxW6" colab_type="code" colab={"autoexec": {"startup": false, "wait_interval": 0}}
import tensorflow as tf            # import tensorflow, our machine learning library
import numpy as np                 # import numpy, our linear algebra library
import matplotlib.pyplot as plt    # import matplotlib, our graph plotting library

from math import ceil,floor        # import round up and round down functions
import helper_funcs as helper

# + [markdown] id="0LdBjfkvSpsD" colab_type="text"
# ## Data

# + id="y5nA3LM4kxXA" colab_type="code" colab={"autoexec": {"startup": false, "wait_interval": 0}, "base_uri": "https://localhost:8080/", "height": 126} outputId="75d5ecdc-6f8a-4e43-e9a0-488cfc7a5151" executionInfo={"status": "ok", "timestamp": 1530976684169, "user_tz": -60, "elapsed": 5636, "user": {"displayName": "<NAME>", "photoUrl": "//lh4.googleusercontent.com/-8IeZZ41ybbo/AAAAAAAAAAI/AAAAAAAACkI/_LkIlaW8I6g/s50-c-k-no/photo.jpg", "userId": "105201401933439284277"}}
# this is the directory where we will keep and external files, eg. data, logs
model_root_dir = '/content/'

# get data
mnist = helper.MNIST_data(model_root_dir+'MNIST_data/')

# + id="TYrM5kF6kxXH" colab_type="code" colab={"autoexec": {"startup": false, "wait_interval": 0}}
image_dims = (28,28)  # the image dimensions
input_size = 28**2    # the unstacked image size
num_classes = 10      # the number of classes to predict

# + [markdown] id="PHTbwzBISs-m" colab_type="text"
# ### Input Data

# + id="qcnZUIMhcw8n" colab_type="code" colab={"autoexec": {"startup": false, "wait_interval": 0}, "base_uri": "https://localhost:8080/", "height": 35} outputId="6268cd7f-6ddb-4a59-d81f-20d4ebf820e2" executionInfo={"status": "ok", "timestamp": 1530976685676, "user_tz": -60, "elapsed": 656, "user": {"displayName": "<NAME>", "photoUrl": "//lh4.googleusercontent.com/-8IeZZ41ybbo/AAAAAAAAAAI/AAAAAAAACkI/_LkIlaW8I6g/s50-c-k-no/photo.jpg", "userId": "105201401933439284277"}}
# this is the shape of our array of training images
# input data arrays = mnist.<dataset>.images
print('training images array:', mnist.train_images.shape)

# + [markdown] id="kFumLIgzSuqh" colab_type="text"
# ### Expected Output Data
#

# + id="SN7IpGEDedoB" colab_type="code" colab={"autoexec": {"startup": false, "wait_interval": 0}, "base_uri": "https://localhost:8080/", "height": 35} outputId="195246a7-5c54-4b6c-ff3d-3ec2a99d10c5" executionInfo={"status": "ok", "timestamp": 1530976686334, "user_tz": -60, "elapsed": 474, "user": {"displayName": "<NAME>", "photoUrl": "//lh4.googleusercontent.com/-8IeZZ41ybbo/AAAAAAAAAAI/AAAAAAAACkI/_LkIlaW8I6g/s50-c-k-no/photo.jpg", "userId": "105201401933439284277"}}
# this is the shape of our array of training labels
# input data arrays = mnist.<dataset>.labels
print('training images array:', mnist.train_labels.shape)

# + [markdown] id="sR2iJ3EWS2WU" colab_type="text"
# ### Data Sets

# + id="E3p4h74ZektW" colab_type="code" colab={"autoexec": {"startup": false, "wait_interval": 0}, "base_uri": "https://localhost:8080/", "height": 72} outputId="8408e026-a310-486a-fd5b-99ab614e934d" executionInfo={"status": "ok", "timestamp": 1530976687129, "user_tz": -60, "elapsed": 701, "user": {"displayName": "<NAME>", "photoUrl": "//lh4.googleusercontent.com/-8IeZZ41ybbo/AAAAAAAAAAI/AAAAAAAACkI/_LkIlaW8I6g/s50-c-k-no/photo.jpg", "userId": "105201401933439284277"}}
# show data sets
print('training data shape', mnist.train_images.shape)
print('validation data shape', mnist.validation_images.shape)
print('testing data shape', mnist.test_images.shape)

# + [markdown] id="72O480LES8xx" colab_type="text"
# ## Hyperparameters

# + id="patpMsApkxXU" colab_type="code" colab={"autoexec": {"startup": false, "wait_interval": 0}}
# Hyperparams

# number of images to feed in at each step
batch_size = 100

# coeficient to our weight update
learning_rate = 0.1

# number of times to feed in the whole dataset
epochs = 2

# the number of iterations = how many times do we need to input a batch to have inputed the whole training dataset
iterations = ceil(mnist.number_train_samples/batch_size)

# number of neurons in hidden layer
hidden_size = 256

# + id="TYWqqn2SkxXX" colab_type="code" colab={"autoexec": {"startup": false, "wait_interval": 0}}
tf.reset_default_graph()

# + [markdown] id="p36-xDFUTNdu" colab_type="text"
# ## Placeholders

# + id="B3ainDiRkxXb" colab_type="code" colab={"autoexec": {"startup": false, "wait_interval": 0}, "base_uri": "https://localhost:8080/", "height": 54} outputId="8d1f67c6-505b-4e0e-805f-0431953671f8" executionInfo={"status": "ok", "timestamp": 1530976689035, "user_tz": -60, "elapsed": 316, "user": {"displayName": "<NAME>", "photoUrl": "//lh4.googleusercontent.com/-8IeZZ41ybbo/AAAAAAAAAAI/AAAAAAAACkI/_LkIlaW8I6g/s50-c-k-no/photo.jpg", "userId": "105201401933439284277"}}
# input placeholders

# input image
# when dimension = None, it means it's undefined, and will take whatever dimension size we give it
X_placeholder = tf.placeholder(shape=[None, input_size], dtype=tf.float32, name='X_placeholder')

# image labels
Y_placeholder = tf.placeholder(shape=[None, num_classes], dtype=tf.int64, name='Y_placeholder')


print(X_placeholder)
print(Y_placeholder)

# + [markdown] id="CUhBri0QTTDC" colab_type="text"
# ## Variables

# + id="igi4r9MpkxXf" colab_type="code" colab={"autoexec": {"startup": false, "wait_interval": 0}, "base_uri": "https://localhost:8080/", "height": 90} outputId="01e225cd-39ac-4224-d5c9-32338b357881" executionInfo={"status": "ok", "timestamp": 1530976689584, "user_tz": -60, "elapsed": 465, "user": {"displayName": "<NAME>", "photoUrl": "//lh4.googleusercontent.com/-8IeZZ41ybbo/AAAAAAAAAAI/AAAAAAAACkI/_LkIlaW8I6g/s50-c-k-no/photo.jpg", "userId": "105201401933439284277"}}
# Variables

# First layer Variables
W1 = tf.Variable(np.random.randn(input_size, hidden_size), dtype=tf.float32, name='W1')
b1 = tf.Variable(np.random.randn(hidden_size), dtype=tf.float32, name='b1')

# Second layer Variables
W2 = tf.Variable(np.random.randn(hidden_size, num_classes), dtype=tf.float32, name='W2')
b2 = tf.Variable(np.random.randn(num_classes), dtype=tf.float32, name='b2')

print(W1)
print(b1)

print(W2)
print(b2)

# + [markdown] id="BhU7AP7cTX9B" colab_type="text"
# ## Forward Pass

# + id="KpRui1fGkxXj" colab_type="code" colab={"autoexec": {"startup": false, "wait_interval": 0}, "base_uri": "https://localhost:8080/", "height": 54} outputId="4238d6bd-d64e-4382-9a99-aa39234a7a71" executionInfo={"status": "ok", "timestamp": 1530976690149, "user_tz": -60, "elapsed": 488, "user": {"displayName": "<NAME>", "photoUrl": "//lh4.googleusercontent.com/-8IeZZ41ybbo/AAAAAAAAAAI/AAAAAAAACkI/_LkIlaW8I6g/s50-c-k-no/photo.jpg", "userId": "105201401933439284277"}}
# first fully connected layer
# output is our hidden layer
# we matrix multiply (tf.matmul)
# add bias (+ b1)
# then apply activation function (tf.nn.relu)
hidden_layer = tf.nn.relu(tf.matmul(X_placeholder, W1) + b1)

# second fully connected layer
# output is our NN output (predictions)
Y_prediction = tf.matmul(hidden_layer, W2) + b2

print(hidden_layer)
print(Y_prediction)

# + [markdown] id="egQwXN9BTmIF" colab_type="text"
# ## Loss

# + id="TytxCw2ckxXs" colab_type="code" colab={"autoexec": {"startup": false, "wait_interval": 0}}
# loss function, softmax and cross entropy

# logits = NN output
# labels = expected outputs
# we take the mean so during gradient descent we take the mean step
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=Y_placeholder, logits=Y_prediction, name='loss'))

# + [markdown] id="7yL4q1GNTv6f" colab_type="text"
# ## Gradient Descent

# + id="kMqQCo4wkxX5" colab_type="code" colab={"autoexec": {"startup": false, "wait_interval": 0}}
# gradient descent optimizer

# we input the learning rate and set it to minimize the loss value
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)

# + [markdown] id="2-8OSQ8uroo3" colab_type="text"
# ## Calculating Accuracy

# + id="JonLecJNkxX8" colab_type="code" colab={"autoexec": {"startup": false, "wait_interval": 0}}
# calculating accuracy

# vector of bools on whether the prediction was right or wrong
correct_prediction = tf.equal(tf.argmax(Y_prediction, 1), tf.argmax(Y_placeholder, 1))
# taking the average of the above converted to binary ints (0 for wrong, 1 for right)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
# get probability output from softmax
predictions = tf.nn.softmax(Y_prediction)

# + [markdown] id="JsggtzSVrr4d" colab_type="text"
# ## Tensorflow Sessions

# + id="dmIQDF7-kxX-" colab_type="code" colab={"autoexec": {"startup": false, "wait_interval": 0}}
# Initialize our session and vars
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())

# + [markdown] id="07_bXP-PUA0U" colab_type="text"
# ## Training

# + id="5EtDwiY_kxYB" colab_type="code" colab={"autoexec": {"startup": false, "wait_interval": 0}, "base_uri": "https://localhost:8080/", "height": 235} outputId="e7e7c857-f417-4185-b859-dd178f7217fe" executionInfo={"status": "ok", "timestamp": 1530976697055, "user_tz": -60, "elapsed": 4254, "user": {"displayName": "<NAME>", "photoUrl": "//lh4.googleusercontent.com/-8IeZZ41ybbo/AAAAAAAAAAI/AAAAAAAACkI/_LkIlaW8I6g/s50-c-k-no/photo.jpg", "userId": "105201401933439284277"}}
# initialize empty array to hold losses for each training step
losses = np.array([])

for epoch in range(epochs):
    print('New epoch', str(epoch+1)+'/'+str(epochs))
    for iteration in range(iterations):

        # get batches of training data for current step
        batch_xs, batch_ys = mnist.get_batch(iteration, batch_size)

        # run training step and get loss values to plot later
        # calling optimizer will run the forward pass, the loss function and the pack pass
        # we call the loss function just so we can record it
        _, _loss = sess.run([optimizer, loss], 
                            feed_dict={
                                X_placeholder: batch_xs,
                                Y_placeholder: batch_ys
                            })

        if (iteration+1) % floor(iterations/5) == 0:

            # get accuracy using validation set
            _accuracy = sess.run(accuracy, feed_dict={X_placeholder : mnist.validation_images,
                                                      Y_placeholder : mnist.validation_labels
                                                     })

            # print current step, batch loss and validation accuracy
            print('step', str(iteration+1)+'/'+str(iterations), 'loss', _loss, 'accuracy', str(round(100*_accuracy,2))+'%')

        # append the loss to the losses array to plot
        losses = np.append(losses, _loss)

# + id="_ftB92TwkxYE" colab_type="code" colab={"autoexec": {"startup": false, "wait_interval": 0}, "base_uri": "https://localhost:8080/", "height": 378} outputId="6a7180c4-ac87-411e-d406-dbeff2c1117d" executionInfo={"status": "ok", "timestamp": 1530976697587, "user_tz": -60, "elapsed": 495, "user": {"displayName": "<NAME>", "photoUrl": "//lh4.googleusercontent.com/-8IeZZ41ybbo/AAAAAAAAAAI/AAAAAAAACkI/_LkIlaW8I6g/s50-c-k-no/photo.jpg", "userId": "105201401933439284277"}}
# plot training losses

fig, ax = plt.subplots(figsize=(10,6))
ax.plot(losses)
ax.grid(True)

# + [markdown] id="ijozOrq2UDXq" colab_type="text"
# ## Testing

# + id="tfaCaDxKkxYJ" colab_type="code" colab={"autoexec": {"startup": false, "wait_interval": 0}, "base_uri": "https://localhost:8080/", "height": 35} outputId="5c40a4b4-cb36-4aa1-f9c1-cd720101c1ff" executionInfo={"status": "ok", "timestamp": 1530976698269, "user_tz": -60, "elapsed": 626, "user": {"displayName": "<NAME>", "photoUrl": "//lh4.googleusercontent.com/-8IeZZ41ybbo/AAAAAAAAAAI/AAAAAAAACkI/_LkIlaW8I6g/s50-c-k-no/photo.jpg", "userId": "105201401933439284277"}}
# find accuracy on test set
_accuracy = sess.run(accuracy, feed_dict={X_placeholder : mnist.test_images,
                                          Y_placeholder : mnist.test_labels
                                         })

print(str(round(100*_accuracy,2))+'%')

# + id="44N-HS2ckxYO" colab_type="code" colab={"autoexec": {"startup": false, "wait_interval": 0}, "base_uri": "https://localhost:8080/", "height": 1077} outputId="c2942c9d-5f73-41af-bf69-9a0ac2d98f8a" executionInfo={"status": "ok", "timestamp": 1530976699221, "user_tz": -60, "elapsed": 890, "user": {"displayName": "<NAME>", "photoUrl": "//lh4.googleusercontent.com/-8IeZZ41ybbo/AAAAAAAAAAI/AAAAAAAACkI/_LkIlaW8I6g/s50-c-k-no/photo.jpg", "userId": "105201401933439284277"}}
# show test images with prediction probabilities

test_idxs = [1,11]

for test_idx in test_idxs:
    # get input image and corresponding label
    test_x, test_y = mnist.test_images[test_idx], mnist.test_labels[test_idx]

    # make prediction
    _prediction = sess.run(predictions, feed_dict={X_placeholder: test_x.reshape(1,-1)})

    # show prediction and true answer
    print('prediction:', np.argmax(_prediction), 'answer:', np.argmax(test_y))

    # show probabilities for all values
    for i,val in enumerate(_prediction[0]):
        print(i, str(round(100*val,3))+'%')

    # plot test image
    plt.imshow(test_x.reshape(image_dims), cmap='binary')
    plt.grid(False)
    plt.show()

# + id="DdVNfsmuCP8H" colab_type="code" colab={"autoexec": {"startup": false, "wait_interval": 0}}


StarcoderdataJupyterScriptsDedupFiltered