# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] colab_type="text" id="view-in-github"
# <a href="https://colab.research.google.com/github/mlvlab/COSE474/blob/master/2_MNIST_Tutorial%20(CNN).ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>
# -

# ## MNIST_Tutorial (CNN) in PyTorch
#
# ### Reference
# * [PyTorch Tutorial MNIST](https://github.com/GunhoChoi/PyTorch-FastCampus/tree/master/03_CNN_Basics/0_MNIST)
# * [VISUALIZING MODELS, DATA, AND TRAINING WITH TENSORBOARD](https://tutorials.pytorch.kr/intermediate/tensorboard_tutorial.html?highlight=mnist)

# ### GOALS
#
# 1. We will make a simple convolutional neural network(CNN).
# 2. Then, we will train the network with MNIST.
# 3. We will learn how to use Tensorboard as well to monitor training.
#
#
#

# ### Settings
# Import required libraries
# Set hyperparameters

# + colab={} colab_type="code" id="CAuEKfCwR8mD"
import torch
import torch.nn as nn
import torch.optim as optim
import torchvision

import torch.nn.init as init
import torchvision.datasets as dset
import torchvision.transforms as transforms
from torch.utils.data import DataLoader

from torch.utils.tensorboard import SummaryWriter

# Writer will output to ./runs/ directory by default
writer = SummaryWriter()

# Setting hyperparameters
batch_size = 256
learning_rate = 0.0002
num_epoch = 10
# -

# ### Data Generation
# Download Data
# Set DataLoader

# + colab={} colab_type="code" id="jfyJcAVnScAB"
# Loading MNIST dataset
mnist_train = dset.MNIST("./", train=True, transform=transforms.ToTensor(),
                         target_transform=None, download=True)
mnist_test = dset.MNIST("./", train=False, transform=transforms.ToTensor(),
                         target_transform=None, download=True)

train_loader = torch.utils.data.DataLoader(mnist_train, batch_size = batch_size,
                                           shuffle = True, num_workers = 2, drop_last = True)
test_loader = torch.utils.data.DataLoader(mnist_test, batch_size = batch_size,
                                           shuffle = False, num_workers = 2, drop_last = True)


# -

# ### Define Convolutional Neural Network
# 1. MNIST data
# 2. 3 convolutional layers
# 3. 2 fully connected layers

# + colab={} colab_type="code" id="MB1MVfIoScDl"
# Creating CNN model
class CNN(nn.Module):
    def __init__(self):
        super(CNN, self).__init__()
        self.layer = nn.Sequential(
            nn.Conv2d(1,16,5),
            nn.ReLU(),
            nn.Conv2d(16,32,5),
            nn.ReLU(),
            nn.MaxPool2d(2,2),
            nn.Conv2d(32,64,5),
            nn.ReLU(),
            nn.MaxPool2d(2,2)        
        )
        self.fc_layer = nn.Sequential(
            nn.Linear(64*3*3,100),
            nn.ReLU(),
            nn.Linear(100,10)
        )
        
    def forward(self, x):
        out = self.layer(x)
        out = out.view(batch_size, -1)
        out = self.fc_layer(out)
        return out


# -

# ### Train & Test

# + colab={} colab_type="code" id="Bjxn_kKuScKj"
# Declare device(GPU/CPU), declare Loss func and optimizer
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model = CNN().to(device)
loss_func = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)

loss_arr = []

train_iter = 0
test_iter = 0
test_acc_iter = 0

# Training
for i in range(num_epoch):
    for j, [image, label] in enumerate(train_loader):
        x = image.to(device)
        y_ = label.to(device)
        
        optimizer.zero_grad()
        output = model.forward(x)
        loss = loss_func(output, y_)
        loss.backward()
        optimizer.step()
            
        if j % 1000 == 0:
            print(loss)
            loss_arr.append(loss.cpu().detach().numpy())
            
            # Tensorboard : train_loss
            writer.add_scalar('Loss/train', loss.item(), train_iter)
            train_iter += 1
                          
correct = 0
total = 0

# Testing
with torch.no_grad():
    for k, [image, label] in enumerate(test_loader):
        x = image.to(device)
        y_ = label.to(device)
        
        output = model.forward(x)
        _, output_index = torch.max(output, 1)
        
        test_loss = loss_func(output, y_)
        
        total += label.size(0)
        correct += (output_index == y_).sum().float()
        
        accuracy = 100*correct/total
        
        # Tensorboard : test_loss
        writer.add_scalar('Loss/test', test_loss.item(), test_iter)
        test_iter += 1
        
        # Tensorboard : test_Accuracy  
        writer.add_scalar('Accuracy/test', accuracy.item(), test_acc_iter)
        test_acc_iter += 1
        
    print("Accuracy of Test Data : {}".format(100*correct/total))

# Tensorboard : image (1 batch)
images, labels = next(iter(train_loader))
grid = torchvision.utils.make_grid(images)
writer.add_image('images', grid, 0)

# Tensorboard : model graph
grp = images.to(device)
writer.add_graph(model, grp)
    
writer.close()
# -

# ### Using Tensorboard to monitor training & testing

# + colab={} colab_type="code" id="J3dHkTdsScNx"
# Install tf-nightly package
# !pip install -q tf-nightly-2.0-preview

# + colab={} colab_type="code" id="j2OTDHp3ScRK"
# %load_ext tensorboard

# + colab={} colab_type="code" id="NZUMYffwScUa"
# Run tensorboard
# if you cannot see anything, just try running one more time
# %tensorboard --logdir=runs

# + colab={} colab_type="code" id="6BkpiNlASnXd"
# you can use linux command like 'ls, cd, cd.., pip, rm'.

# + colab={} colab_type="code" id="8rCUxSCwSnas"
# (option) remove 'runs' directory
# !rm -rf runs

# + colab={} colab_type="code" id="v_egrnirSndp"
# (option) list files to check 'runs' directory
# ls

# + colab={} colab_type="code" id="dRvcarEQSngv"


StarcoderdataJupyterScriptsDedupFiltered