# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python [conda env:tensorflow]
#     language: python
#     name: conda-env-tensorflow-py
# ---

# # CIFAR-10
#
# Link: https://www.cs.toronto.edu/~kriz/cifar.html
#
# The CIFAR-10 dataset consists of 60000 32x32 colour images in 10 classes, with 6000 images per class. There are 50000 training images and 10000 test images. 
#
# The classes are completely mutually exclusive. There is no overlap between automobiles and trucks. "Automobile" includes sedans, SUVs, things of that sort. "Truck" includes only big trucks. Neither includes pickup trucks.

# all the imports
import tensorflow as tf
import numpy as np
import glob

# ## Downloading and Preparing the Dataset
# The archive contains the files data_batch_1, data_batch_2, ..., data_batch_5, as well as test_batch. Each of these files is a Python "pickled" object produced with Pickle.
#
# Loaded in this way, each of the batch files contains a dictionary with the following elements:
# - data -- a 10000x3072 numpy array of uint8s. Each row of the array stores a 32x32 colour image. The first 1024 entries contain the red channel values, the next 1024 the green, and the final 1024 the blue. The image is stored in row-major order, so that the first 32 entries of the array are the red channel values of the first row of the image.
# - labels -- a list of 10000 numbers in the range 0-9. The number at index i indicates the label of the ith image in the array data.

# get all the dataset file names
filenames_train = np.array(glob.glob("./cifar-10-batches-py/data_batch_*"), dtype=str)
filename_test = "./cifar-10-batches-py/test_batch"
filename_label = "./cifar-10-batches-py/batches.meta"

filenames_train, filename_test, filename_label


# unpickle the data and return the dictionary containing the data and label
def unpickle(file):
    import pickle
    with open(file, 'rb') as f:
        dict = pickle.load(f, encoding='bytes')
    return dict


train_images = []
labels = []

for file in filenames_train:
    cifar = unpickle(file)
    train_images.extend(cifar[b'data'])
    labels += cifar[b'labels']

X_train = np.array(train_images, dtype=float)
X_train = X_train.reshape(-1, 32, 32, 3)
X_train.shape

_y_train = np.array(labels).reshape(-1,1)
y_train = np.zeros((50000,10))
for i in range(50000):
    y_train[i][_y_train[i]] = 1
y_train.shape

X_train[94], y_train[:10]

test_images = []
test_labels = []

cifar = unpickle(filename_test)
test_images = cifar[b'data']
test_labels = cifar[b'labels']

X_test = np.array(test_images, dtype=float)
X_test = X_test.reshape(-1, 32, 32, 3)
X_test.shape

_y_test = np.array(test_labels).reshape(-1,1)
y_test = np.zeros((10000,10))
for i in range(10000):
    y_test[i][_y_test[i]] = 1
y_test.shape

X_test[94], y_test[94]

# ## Create a tensorflow Input pipeline

n_batch = 512
buffer_size = 1024


# create the image parser function
def _parser_function(image, label):
    image_norm = tf.divide(image, tf.convert_to_tensor(255.0, dtype=tf.float64))
    return image_norm, label


# Input Pipeline

# create input pipeline using tf.data
dataset = tf.data.Dataset.from_tensor_slices((X_train, y_train))
dataset = dataset.shuffle(buffer_size=buffer_size)
dataset = dataset.apply(tf.contrib.data.map_and_batch(map_func=_parser_function, batch_size=n_batch))
iterator = dataset.make_initializable_iterator()

# create batch iterators
input_mini_batch, label_mini_batch = iterator.get_next()

# Test Pipeline

# create test pipeline using tf.data
test_dataset = tf.data.Dataset.from_tensor_slices((X_test, y_test))
test_dataset = test_dataset.shuffle(buffer_size=buffer_size)
test_dataset = test_dataset.apply(tf.contrib.data.map_and_batch(map_func=_parser_function, batch_size=n_batch))
test_iterator = test_dataset.make_initializable_iterator()

# create batch iterators
X_test_mini_batch, y_test_mini_batch = test_iterator.get_next()

# ## Define CNN Model
# We'll use a simple 2 layered CNN model for this approach

X = tf.placeholder(tf.float32, [None, 32, 32, 3])
y_ = tf.placeholder(tf.float32, [None, 10])


# +
# create weights and biases with a small amount of noise
def weight_variable(shape):
    initial = tf.truncated_normal(shape, stddev=0.1)
    return tf.Variable(initial)

def bias_variable(shape):
    initial = tf.constant(0.1, shape=shape)
    return tf.Variable(initial)


# +
# convolution and pooling layers
def conv2d(x, W):
    return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')

def max_pool_2x2(x):
    return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')


# +
# define weights and biases
W_conv1 = weight_variable([5,5,3,64])
b_conv1 = bias_variable([64])

W_conv2 = weight_variable([5,5,64,128])
b_conv2 = bias_variable([128])

# fully connected layers
W_fc1 = weight_variable([8192,2048])
b_fc1 = bias_variable([2048])

W_fc2 = weight_variable([2048,1024])
b_fc2 = bias_variable([1024])

W_fc3 = weight_variable([1024,10])
b_fc3 = bias_variable([10])

# +
# create CNN tensorflow model
conv1 = conv2d(X, W_conv1)
conv1_activated = tf.nn.relu(conv1 + b_conv1)
conv1_maxpool = max_pool_2x2(conv1_activated)

conv2 = conv2d(conv1_maxpool, W_conv2)
conv2_activated = tf.nn.relu(conv2 + b_conv2)
conv2_maxpool = max_pool_2x2(conv2_activated)
print(conv2_maxpool)
# -

# flatten output from conv layers
x_fc = tf.reshape(conv2_maxpool, shape=[-1, 8*8*128])

# +
# fully conected layer 1
fc1 = tf.add(tf.matmul(x_fc, W_fc1), b_fc1)
fc1_activated = tf.nn.relu(fc1)

# fully conected layer 2
fc2 = tf.add(tf.matmul(fc1_activated, W_fc2), b_fc2)
fc2_activated = tf.nn.relu(fc2)

# fully conected layer 3
y = tf.add(tf.matmul(fc2_activated, W_fc3), b_fc3)
# -

# ## Cost function
# We'll use softmax as our cost function

cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=y, labels=y_))

# ## Optimizer
# We'll use Adam Optimizer for testing purposes

optim = tf.train.AdamOptimizer()

# ### Now let's minimize the cost using our optimizer

train = optim.minimize(cost)

# ### Setup graph to calculate accuracy

# returns a boolean array
predictions = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))

# cast the boolean array to float and calculate accuracy
accuracy = tf.reduce_mean(tf.cast(predictions, tf.float32))

# ## Create a tensorflow graph
# init the tensorflow session and create train graph

# initialize tensorflow interactive session#
sess = tf.InteractiveSession()

# initialize variables inside the graph
tf.global_variables_initializer().run()


# ### Load Graph

# + active=""
# # load graph
# saver = tf.train.Saver()
# saver.restore(sess, "./model.ckpt")
# -

# ### Save Graph

def save_graph(epoch):
    saver = tf.train.Saver()
    save_path = saver.save(sess, "./model_epoch_final.ckpt", global_step=epoch)


# Now let's create a function to calculate accuracy of our model

def calculate_accuracy():
    # reset iterators
    sess.run(iterator.initializer)
    sess.run(test_iterator.initializer)
    
    train_acc = 0
    train_iters = 0
    
    # check train accuracy over 5 mini batches, to reduce computation
    for _ in range(10):
        train_iters += 1
        X_test, y_test = sess.run([input_mini_batch, label_mini_batch])
        train_acc += sess.run(accuracy, feed_dict={X: X_test, y_: y_test})

    test_acc = 0
    test_iters = 0
    while True:
        try:
            test_iters += 1
            X_test, y_test = sess.run([X_test_mini_batch, y_test_mini_batch])
            test_acc += sess.run(accuracy, feed_dict={X: X_test, y_: y_test})
        except tf.errors.OutOfRangeError:
            break
    
    # print accuracy
    print(f"Train accuracy: {train_acc/train_iters}")
    print(f"Test accuracy: {test_acc/test_iters}")


# Finally it's time to train our model

n_epochs = 20

for epoch in range(0,n_epochs):
    epoch_loss = 0
    sess.run(iterator.initializer)
    
    while True:
        try:
            X_for_train, y_for_train = sess.run([input_mini_batch, label_mini_batch])
            t, c = sess.run([train, cost], feed_dict={X: X_for_train, y_: y_for_train})
            epoch_loss += c
        except tf.errors.OutOfRangeError:
            break

    # print loss
    print(f"Epoch {epoch} out of {n_epochs}, loss: {epoch_loss}")
    
    # print train and test accuracies
    calculate_accuracy()
    
    # Save Graph
    if epoch % 10 == 0 and epoch >= 2:
        save_graph(epoch)

StarcoderdataJupyterScriptsDedupFiltered