# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + slideshow={"slide_type": "slide"}
import matplotlib.pyplot as plt
import numpy as np
import os
import tarfile
import tensorflow as tf

from collections import Counter
from scipy import misc
from PIL import Image
from sklearn.metrics import confusion_matrix

# + [markdown] slideshow={"slide_type": "slide"}
# #### Dense layer limitations
#
# * pixel values are individual features, no neighboring information is stored
# * number of weigths for large images can be very large

# + slideshow={"slide_type": "slide"}
model = tf.keras.models.Sequential([
    tf.keras.layers.Flatten(input_shape=(28, 28)),
    tf.keras.layers.Dense(64, activation=tf.nn.relu),
    tf.keras.layers.Dense(32, activation=tf.nn.relu),
    tf.keras.layers.Dense(10, activation=tf.nn.softmax),
])

# + slideshow={"slide_type": "slide"}
model.summary()

# + slideshow={"slide_type": "slide"}
# Same network for larger input images 
model = tf.keras.models.Sequential([
    tf.keras.layers.Flatten(input_shape=(512, 512)),
    tf.keras.layers.Dense(64, activation=tf.nn.relu),
    tf.keras.layers.Dense(32, activation=tf.nn.relu),
    tf.keras.layers.Dense(10, activation=tf.nn.softmax),
])

# + slideshow={"slide_type": "slide"}
model.summary()

# + slideshow={"slide_type": "slide"}
tf.keras.backend.clear_session()

# + [markdown] slideshow={"slide_type": "slide"}
# ## Convolution with filters

# + slideshow={"slide_type": "slide"}
test_image = misc.ascent()

fig = plt.figure(figsize=(6, 6))
plt.imshow(test_image, cmap='gray')
plt.show()


# + [markdown] slideshow={"slide_type": "slide"}
# Convolution with a $3\times 3$ filter
#
# $$ A \circledast F = 
# \begin{bmatrix} 
#     2 & 0 & 1 & 7 & 2 \\ 
#     0 & 0 & 1 & 1 & 1 \\ 
#     0 & 3 & 2 & 8 & 3 \\ 
#     1 & 1 & 4 & 3 & 4 \\
#     0 & 0 & 1 & 6 & 9 \\
# \end{bmatrix} \circledast
# \begin{bmatrix}
# 1 & 0 & -1 \\ 1 & 0 & -1 \\ 1 & 0 & -1 
# \end{bmatrix} =
# \begin{bmatrix}
# -2 & -13 & -2 \\
# -6 & -8 & -1 \\
# -6 & -13 & -9\\
# \end{bmatrix}
# $$

# + slideshow={"slide_type": "slide"}
def convolution(matrix, filt):
    r, c = matrix.shape
    f = len(filt)
    result = np.zeros((r-f+1, c-f+1))
    for i in range(r-f+1):
        for j in range(c-f+1):
            result[i, j] = np.sum(np.multiply(matrix[i:i+f, j:j+f], filt))
    return result


# + slideshow={"slide_type": "slide"}
filters = [
    [[1, 0, -1], [1, 0, -1], [1, 0, -1]],   # vertical edge detector
    [[1, 1, 1], [0, 0, 0], [-1, -1, -1]],   # horizontal edge detector
    [[1, 0, -1], [2, 0, -2], [1, 0, -1]]    # Sobel filter - edge detection
]


# filters = [
#     [[1, 0, -1], [1, 0, -1], [1, 0, -1]],
#     [[0, -1, 0], [-1, 5, -1], [0, -1, 0]],   # what is this?
#     [[1/9, 1/9, 1/9], [1/9, 1/9, 1/9], [1/9, 1/9, 1/9]]   # what is this?
# ]

# + slideshow={"slide_type": "slide"}
filtered_images = []
for filt in filters:
    filtered_image = convolution(test_image, filt)
    filtered_image[filtered_image < 0] = 0
    filtered_image[filtered_image > 255] = 255
    filtered_images.append(filtered_image)

# + slideshow={"slide_type": "slide"}
fig = plt.figure(figsize=(14, 14))
fig.add_subplot(2, 2, 1)
plt.imshow(test_image, cmap='gray')
fig.add_subplot(2, 2, 2)
plt.imshow(filtered_images[0], cmap='gray')
fig.add_subplot(2, 2, 3)
plt.imshow(filtered_images[1], cmap='gray')
fig.add_subplot(2, 2, 4)
plt.imshow(filtered_images[2], cmap='gray')
plt.show()


# + slideshow={"slide_type": "slide"}
def maxpooling(matrix):
    r, c = matrix.shape
    pooled_rows, pooled_cols = r // 2, c // 2
    result = np.empty(shape=(pooled_rows, pooled_cols), dtype=matrix.dtype)
    for ix in range(pooled_rows):
        for jy in range(pooled_cols):
            result[ix, jy] = np.max(matrix[2*ix:2*ix+2, 2*jy:2*jy+2])
    return result


# + slideshow={"slide_type": "slide"}
pooled_images = [maxpooling(image) for image in filtered_images]

# + slideshow={"slide_type": "slide"}
fig = plt.figure(figsize=(16, 6))
fig.add_subplot(1, 3, 1)
plt.imshow(test_image, cmap='gray')
fig.add_subplot(1, 3, 2)
plt.imshow(filtered_images[0], cmap='gray')
fig.add_subplot(1, 3, 3)
plt.imshow(pooled_images[0], cmap='gray')
plt.show()

# + [markdown] slideshow={"slide_type": "slide"}
# ## CIFAR-10

# + slideshow={"slide_type": "slide"}
tf.config.list_physical_devices('GPU')

# + slideshow={"slide_type": "fragment"}
# !nvidia-smi

# + slideshow={"slide_type": "slide"}
# (X_train, y_train), (X_test, y_test) = tf.keras.datasets.cifar10.load_data()

# + slideshow={"slide_type": "slide"}
# # !wget http://pjreddie.com/media/files/cifar.tgz -O /tmp/cifar.tgz
#
with tarfile.open("data/cifar.tgz","r") as tar:
    tar.extractall("/tmp")

# + slideshow={"slide_type": "slide"}
folder = "/tmp/cifar"


def get_filenames(folder):
    return [f.name for f in os.scandir(folder) if f.is_file()]


def read_cifar_dataset(folder):
    training_folder = f"{folder}/train"
    test_folder = f"{folder}/test"
    X_train, y_train = read_images_in_folder(training_folder)
    X_test, y_test = read_images_in_folder(test_folder)
    return (X_train, y_train), (X_test, y_test)


def read_images_in_folder(folder):
    filenames = get_filenames(folder)
    images = []
    labels = []
    for filename in filenames:
        path = f"{folder}/{filename}"
        name, _ = os.path.splitext(path)
        label = name.split("_")[-1]
        image = np.array(Image.open(path)) / 255.0
        images.append(image)
        labels.append(label)
    return np.array(images), np.array(labels)


# + slideshow={"slide_type": "slide"}
(X_train, y_train), (X_test, y_test) = read_cifar_dataset(folder)

# + slideshow={"slide_type": "slide"}
X_train.shape

# + slideshow={"slide_type": "slide"}
print("Training examples:", X_train.shape, y_train.shape)
print("Test examples:", X_test.shape, y_test.shape)


# + slideshow={"slide_type": "slide"}
def print_image_for_each_label(X, y):
    fig = plt.figure(figsize=(16, 6))
    labels = np.unique(y)
    for p, label in enumerate(labels):
        ix = np.random.choice(np.where(y==label)[0])
        image = X[ix, :, :, :]
        ax = fig.add_subplot(2, 5, p+1)
        plt.imshow(image, cmap=plt.cm.binary)
        ax.set_title(label)
    plt.show()


# + slideshow={"slide_type": "slide"}
print_image_for_each_label(X_train, y_train)


# + slideshow={"slide_type": "slide"}
def categorical_to_numeric(y):
    _, indices = np.unique(y, return_inverse=True)
    return indices


# + slideshow={"slide_type": "slide"}
y_train_num = categorical_to_numeric(y_train)
y_test_num = categorical_to_numeric(y_test)

# + [markdown] slideshow={"slide_type": "slide"}
# #### Logistic regression

# + slideshow={"slide_type": "slide"}
tf.keras.backend.clear_session()

# + slideshow={"slide_type": "slide"}
model = tf.keras.models.Sequential([
    tf.keras.layers.Flatten(input_shape=(32, 32, 3)),
    tf.keras.layers.Dense(10, activation=tf.nn.softmax),
])

# + slideshow={"slide_type": "slide"}
model.summary()

# + slideshow={"slide_type": "slide"}
model.compile(optimizer='adam',
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])

# + slideshow={"slide_type": "slide"}
history = model.fit(X_train, y_train_num, epochs=50, verbose=1, validation_data=(X_test, y_test_num))

# + slideshow={"slide_type": "slide"}
acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']

epochs = np.arange(len(acc))

# + slideshow={"slide_type": "slide"}
plt.figure(figsize=(16, 4))
plt.plot(epochs, acc)
plt.plot(epochs, val_acc)
plt.title('Training and validation accuracy')
plt.show()

# + slideshow={"slide_type": "slide"}
plt.figure(figsize=(16, 4))
plt.plot(epochs, loss)
plt.plot(epochs, val_loss)
plt.title('Training and validation loss')
plt.show()

# + slideshow={"slide_type": "slide"}
y_hat = np.argmax(model.predict(X_test), axis=1)

# + slideshow={"slide_type": "slide"}
plt.figure(figsize=(7, 6))
plt.title('Confusion matrix', fontsize=14)
plt.imshow(confusion_matrix(y_test_num, y_hat))
plt.xticks(np.arange(10), list(range(10)), fontsize=12)
plt.yticks(np.arange(10), list(range(10)), fontsize=12)
plt.colorbar()
plt.show()
print("Test accuracy:", np.equal(y_hat, y_test_num).sum() / len(y_test))

# + slideshow={"slide_type": "slide"}
tf.keras.backend.clear_session()

# + [markdown] slideshow={"slide_type": "slide"}
# #### Fully connected network with hidden layers

# + slideshow={"slide_type": "slide"}
model = tf.keras.models.Sequential([
    tf.keras.layers.Flatten(input_shape=(32, 32, 3)),
    tf.keras.layers.Dense(64, activation=tf.nn.relu),
    tf.keras.layers.Dense(32, activation=tf.nn.relu),
    tf.keras.layers.Dense(10, activation=tf.nn.softmax),
])

# + slideshow={"slide_type": "slide"}
model.summary()

# + slideshow={"slide_type": "slide"}
model.compile(optimizer='adam',
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])

# + slideshow={"slide_type": "slide"}
history = model.fit(X_train, y_train_num, epochs=50, verbose=1, validation_data=(X_test, y_test_num))

# + slideshow={"slide_type": "slide"}
acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']

epochs = np.arange(len(acc))

# + slideshow={"slide_type": "slide"}
plt.figure(figsize=(16, 4))
plt.plot(epochs, acc)
plt.plot(epochs, val_acc)
plt.title('Training and validation accuracy')
plt.show()

# + slideshow={"slide_type": "slide"}
plt.figure(figsize=(16, 4))
plt.plot(epochs, loss)
plt.plot(epochs, val_loss)
plt.title('Training and validation loss')
plt.show()

# + slideshow={"slide_type": "slide"}
y_hat = np.argmax(model.predict(X_test), axis=1)

# + slideshow={"slide_type": "slide"}
plt.figure(figsize=(7, 6))
plt.set_cmap('viridis')
plt.title('Confusion matrix', fontsize=14)
plt.imshow(confusion_matrix(y_test_num, y_hat))
plt.xticks(np.arange(10), list(range(10)), fontsize=12)
plt.yticks(np.arange(10), list(range(10)), fontsize=12)
plt.colorbar()
plt.show()
print("Test accuracy:", np.equal(y_hat, y_test_num).sum() / len(y_test))

# + [markdown] slideshow={"slide_type": "slide"}
# #### CNN - LeNet5

# + slideshow={"slide_type": "slide"}
tf.keras.backend.clear_session()

# + [markdown] slideshow={"slide_type": "slide"}
# ![](images/lenet5.png)

# + slideshow={"slide_type": "slide"}
model = tf.keras.models.Sequential([
    tf.keras.layers.Conv2D(6, kernel_size=5, strides=1, padding='valid', activation='tanh', 
                           input_shape=(32, 32, 3)),
    tf.keras.layers.MaxPooling2D(pool_size=2, strides=2),
    tf.keras.layers.Conv2D(16, kernel_size=5, strides=1, padding='valid', activation='tanh'),
    tf.keras.layers.MaxPooling2D(pool_size=2, strides=2),
    tf.keras.layers.Flatten(),
    tf.keras.layers.Dense(120, activation=tf.nn.tanh),
    tf.keras.layers.Dense(84, activation=tf.nn.tanh),
    tf.keras.layers.Dense(10, activation=tf.nn.softmax),
])

# + slideshow={"slide_type": "slide"}
model.summary()

# + slideshow={"slide_type": "slide"}
model.compile(optimizer='adam',
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])

# + slideshow={"slide_type": "slide"}
history = model.fit(X_train, y_train_num, epochs=50, verbose=1, validation_data=(X_test, y_test_num))

# + slideshow={"slide_type": "slide"}
acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']

epochs = np.arange(len(acc))

# + slideshow={"slide_type": "slide"}
plt.figure(figsize=(16, 4))
plt.plot(epochs, acc)
plt.plot(epochs, val_acc)
plt.title('Training and validation accuracy')
plt.show()

# + slideshow={"slide_type": "slide"}
plt.figure(figsize=(16, 4))
plt.plot(epochs, loss)
plt.plot(epochs, val_loss)
plt.title('Training and validation loss')
plt.show()

# + slideshow={"slide_type": "slide"}
y_hat = np.argmax(model.predict(X_test), axis=1)

# + slideshow={"slide_type": "slide"}
plt.figure(figsize=(7, 6))
plt.title('Confusion matrix', fontsize=14)
plt.imshow(confusion_matrix(y_test_num, y_hat))
plt.xticks(np.arange(10), list(range(10)), fontsize=12)
plt.yticks(np.arange(10), list(range(10)), fontsize=12)
plt.colorbar()
plt.show()
print("Test accuracy:", np.equal(y_hat, y_test_num).sum() / len(y_test))

# + [markdown] slideshow={"slide_type": "slide"}
# #### LeNet5 with regularization

# + slideshow={"slide_type": "slide"}
tf.keras.backend.clear_session()

# + slideshow={"slide_type": "slide"}
model = tf.keras.models.Sequential([
    tf.keras.layers.Conv2D(6, kernel_size=5, strides=1, padding="valid", activation="tanh", 
                           input_shape=(32, 32, 3)),
    tf.keras.layers.MaxPooling2D(pool_size=2, strides=2),
    tf.keras.layers.Dropout(0.15),
    tf.keras.layers.Conv2D(16, kernel_size=5, strides=1, padding="valid", activation="tanh"),
    tf.keras.layers.MaxPooling2D(pool_size=2, strides=2),
    tf.keras.layers.Dropout(0.15),
    tf.keras.layers.Flatten(),
    tf.keras.layers.Dense(120, activation=tf.nn.tanh, kernel_regularizer="l2"),
    tf.keras.layers.Dense(84, activation=tf.nn.tanh, kernel_regularizer="l2"),
    tf.keras.layers.Dense(10, activation=tf.nn.softmax),
])

# + slideshow={"slide_type": "slide"}
model.compile(optimizer='adam',
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])

# + slideshow={"slide_type": "slide"}
history = model.fit(X_train, y_train_num, epochs=30, verbose=1, validation_data=(X_test, y_test_num))

# + slideshow={"slide_type": "slide"}
acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']

epochs = np.arange(len(acc))

# + slideshow={"slide_type": "slide"}
plt.figure(figsize=(16, 4))
plt.plot(epochs, acc)
plt.plot(epochs, val_acc)
plt.title('Training and validation accuracy')
plt.show()

# + slideshow={"slide_type": "slide"}
plt.figure(figsize=(16, 4))
plt.plot(epochs, loss)
plt.plot(epochs, val_loss)
plt.title('Training and validation loss')
plt.show()

# + slideshow={"slide_type": "slide"}
y_hat = np.argmax(model.predict(X_test), axis=1)

# + slideshow={"slide_type": "slide"}
plt.figure(figsize=(7, 6))
plt.title('Confusion matrix', fontsize=14)
plt.imshow(confusion_matrix(y_test_num, y_hat))
plt.xticks(np.arange(10), list(range(10)), fontsize=12)
plt.yticks(np.arange(10), list(range(10)), fontsize=12)
plt.colorbar()
plt.show()
print("Test accuracy:", np.equal(y_hat, y_test_num).sum() / len(y_test))

StarcoderdataJupyterScriptsDedupFiltered