<jupyter_start><jupyter_text>Transfer learn SqueezeNet to four-class face recognition  Make sure the hardware is in order<jupyter_code>gpu_info = !nvidia-smi
gpu_info = '\n'.join(gpu_info)
if gpu_info.find('failed') >= 0:
  print('Select the Runtime > "Change runtime type" menu to enable a GPU accelerator, ')
  print('and then re-execute this cell.')
else:
  print(gpu_info)<jupyter_output>Fri Nov 13 18:03:29 2020       
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 455.32.00    Driver Version: 418.67       CUDA Version: 10.1     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  Tesla V100-SXM2...  Off  | 00000000:00:04.0 Off |                    0 |
| N/A   34C    P0    24W / 300W |      0MiB / 16130MiB |      0%      Default |
|                               |                      |                 ERR! |
+-------------------------------+----------------------+----------------------+
                                                                               
+-------[...]<jupyter_text>Imports<jupyter_code>import time
import os
import copy
import sys

import numpy as np
import matplotlib.pyplot as plt

from tqdm import tqdm
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.data

import torchvision
from torchvision import datasets, models, transforms

from google.cloud import storage

# Placeholder to make it run until the real WoodNet is defined
class WoodNet:
    pass
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
device<jupyter_output><empty_output><jupyter_text>Fetch and extract the data from the storage bucket<jupyter_code># Define paths separate from the heavy operations below
BASE_PATH = "/content"

BLOB_NAME = "faces/balanced_sampled_224px_color_156240_images_70_15_15_split.zip"
zipfilename = os.path.join(BASE_PATH, BLOB_NAME)
extract_to_dir = os.path.join(BASE_PATH, *BLOB_NAME.split(os.path.sep)[:-1])
# Fetch the data
from google.cloud import storage

# Make the required directories
os.makedirs(os.path.join(BASE_PATH, "faces"), exist_ok=True)
os.makedirs(os.path.join(BASE_PATH, "checkpoints"), exist_ok=True)
os.makedirs(os.path.join(BASE_PATH, "logs"), exist_ok=True)

with open(zipfilename, "wb") as f:
    storage.Client.create_anonymous_client().download_blob_to_file(f"gs://tdt4173-datasets/{BLOB_NAME}", f)
# Extract the data
import zipfile

with zipfile.ZipFile(zipfilename, 'r') as zip_ref:
    zip_ref.extractall(extract_to_dir)<jupyter_output><empty_output><jupyter_text>Load the data into wrapper classes and apply normalization<jupyter_code>BATCH_SIZE = 16

data_transforms = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
])

data_dir = os.path.join(extract_to_dir, "sampled_dataset_balanced_244")

image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x),
                                          data_transforms)
                  for x in ['train', 'val', 'test']}

dataloaders = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=BATCH_SIZE,
                                             shuffle=True, num_workers=4)
              for x in ['train', 'val', 'test']}
dataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'val', 'test']}
class_names = image_datasets['train'].classes
print(class_names)
print(image_datasets['val'].classes)
print(dataset_sizes)<jupyter_output>['Kjartan', 'Lars', 'Morgan', 'Other']
['Kjartan', 'Lars', 'Morgan', 'Other']
{'train': 109368, 'val': 23437, 'test': 23436}<jupyter_text>Create a helper function to aid in image plotting and show a random sample of the input data<jupyter_code>def imshow(inp, title=None):
    """Imshow for Tensor."""
    inp = inp.numpy().transpose((1, 2, 0))
    mean = np.array([0.485, 0.456, 0.406])
    std = np.array([0.229, 0.224, 0.225])
    inp = std * inp + mean
    inp = np.clip(inp, 0, 1)
    plt.imshow(inp)
    if title is not None:
        plt.title(title)
    plt.pause(0.001)

# Get a batch of training data
inputs, classes = next(iter(dataloaders['val']))

print(inputs.shape)

# Make a grid from batch
out = torchvision.utils.make_grid(inputs)

imshow(out, title=[class_names[x] for x in classes])<jupyter_output>torch.Size([16, 3, 224, 224])<jupyter_text>Create a function for training and validationThe following function trains the supplied model with the loss criterion and optimizer supplied, for the specified number of epochs. During training it logs the loss and accuracy for both training and validation. Whenever a better model is found on the validation set, the function saves the model parameters to a file for use for inference later.<jupyter_code>def train_model(model, criterion, optimizer, num_epochs=25):
    since = time.time()

    modelname = f"{type(model).__name__}-{since}"
    print(f"Training model: `{type(model).__name__}`")

    best_model_wts = copy.deepcopy(model.state_dict())
    best_acc = 0.0
    num_img = {
        "train": 0,
        "val": 0,
    }
    
    datapoints_per_epoch = 100

    imgs_per_datapoint = {
        "train": int(float(dataset_sizes["train"] / datapoints_per_epoch)),
        "val": int(float(dataset_sizes["val"] / datapoints_per_epoch)),
    }

    for epoch in range(num_epochs):
        print(f"Epoch {epoch}/{num_epochs - 1}")
        print("-" * 10)
        
        with open(os.path.join(BASE_PATH, f"logs/{modelname}.csv"), "a") as f:

            # For each epoch we want to both train and evaluate in that order
            for phase in ["train", "val"]:
                if phase == "train":
                    # Makes the network ready for training, i.e. the parameters can be tuned
                    # and possible Dropouts are activated
                    model.train()
                else:
                    # Makes the network ready for inference, i.e. it is not tunable and will
                    # turn off regularization that might interfere with training
                    model.eval()

                running_loss = 0.0
                running_corrects = 0

                plot_loss = 0
                plot_corrects = 0

                # Iterate over training or validation data
                for inputs, labels in tqdm(dataloaders[phase], desc=f"Epoch: {epoch} ({phase})", file=sys.stdout):
                    inputs = inputs.to(device)
                    labels = labels.to(device)

                    # Reset the gradients before calculating new ones
                    optimizer.zero_grad()

                    
                    # Ask PyTorch to generate computation graph only if in training mode
                    with torch.set_grad_enabled(phase == 'train'):
                        outputs = model(inputs)
                        _, preds = torch.max(outputs, 1)
                        loss = criterion(outputs, labels)
                        
                        # Only perform update steps if we're training
                        if phase == 'train':
                            loss.backward()
                            optimizer.step()


                    # Save values for statistics and logging
                    running_loss += loss.item() * inputs.size(0)
                    running_corrects += torch.sum(preds == labels.data)
                    
                    plot_loss += loss.item() * inputs.size(0)
                    plot_corrects += torch.sum(preds == labels.data)
                    
                    num_img[phase] += BATCH_SIZE
                    
                    if num_img[phase] % imgs_per_datapoint[phase] == 0:
                        f.write(f"{time.time()},{epoch},{phase},\
                        {num_img[phase]},{plot_loss / float(imgs_per_datapoint[phase])},\
                        {plot_corrects / float(imgs_per_datapoint[phase])}\n")
                        
                        plot_loss = 0
                        plot_corrects = 0

                epoch_loss = running_loss / dataset_sizes[phase]
                epoch_acc = running_corrects.double() / dataset_sizes[phase]

                print(f"{phase} Loss: {epoch_loss:.4f} Acc: {epoch_acc:.4f}")

                # deep copy the model
                if phase == "val" and epoch_acc > best_acc:
                    best_acc = epoch_acc
                    best_model_wts = copy.deepcopy(model.state_dict())
                    torch.save(
                        {
                            "loss": epoch_loss,
                            "acc": epoch_acc,
                            "epoch": epoch,
                            "parameters": best_model_wts,
                        },
                        os.path.join(BASE_PATH, f"checkpoints/{modelname}.data"),
                    )
        print()

    time_elapsed = time.time() - since
    print(f"Training complete in {time_elapsed // 60:.0f}m {time_elapsed % 60:.0f}s")
    print(f"Best val Acc: {best_acc:4f}")

    # load best model weights
    model.load_state_dict(best_model_wts)
    return model<jupyter_output><empty_output><jupyter_text>Prepare the home-made CNN – WoodNetBelow is two networks. The first is made by the authors, and is made to be trained from scratch on the training data. The other is fully trained on ImageNet (1000 classes) and fine-tuned on the training data.<jupyter_code>class WoodNet(nn.Module):
    size_after_conv = 7 * 7 * 64
    def __init__(self):
        super(WoodNet, self).__init__()
        self.features = nn.Sequential(   
            nn.Conv2d(3, 32, kernel_size=3, padding=1),
            nn.MaxPool2d(2),
            nn.ReLU(),

            nn.Conv2d(32, 64, kernel_size=3, padding=1),
            nn.MaxPool2d(2),
            nn.ReLU(),
            
            nn.Conv2d(64, 64, kernel_size=3, padding=1),
            nn.MaxPool2d(2),
            nn.ReLU(),
            
            nn.Conv2d(64, 64, kernel_size=3, padding=1),
            nn.MaxPool2d(2),
            nn.ReLU(),
            
            nn.Conv2d(64, 64, kernel_size=3, padding=1),
            nn.MaxPool2d(2),
            nn.ReLU(),
        )
        self.classify = nn.Sequential(
            nn.Linear(self.size_after_conv, 2048),
            nn.ReLU(),
            nn.Linear(2048, 1024),
            nn.ReLU(),
            nn.Dropout(),
            nn.Linear(1024, len(class_names)),
        )

    def forward(self, x):
        x = self.features(x)
        x = x.view(-1, self.size_after_conv)
        x = self.classify(x)

        return x

woodnet = WoodNet().to(device)
print(woodnet)<jupyter_output>WoodNet(
  (features): Sequential(
    (0): Conv2d(3, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (1): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    (2): ReLU()
    (3): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (4): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    (5): ReLU()
    (6): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (7): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    (8): ReLU()
    (9): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (10): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    (11): ReLU()
    (12): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (13): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    (14): ReLU()
  )
  (classify): Sequential(
    (0): Linear(in_features=3136, out_features=2048, bia[...]<jupyter_text>Prepare the pretrained CNN – SqueezeNetBelow is the code for loading in the pretrained SqueezeNet. After it is loaded, the last classification layer is replaced with a one with the correct amount of output classes.<jupyter_code>squeezenet = models.squeezenet1_1(pretrained=True, progress=True)
num_ftr = squeezenet.classifier[1].in_channels
squeezenet.classifier[1] = nn.Conv2d(num_ftr, len(class_names), 1, 1)
squeezenet = squeezenet.to(device)
squeezenet<jupyter_output><empty_output><jupyter_text>Train the networkBelow is code that instantiates the loss function and optimization method and starts the training.To train every parameter in SqueezeNet, set `train_full_network = True`, and to `False` if only the last layer is to be trained.<jupyter_code># network = squeezenet
network = woodnet
train_full_network = False

if train_full_network or isinstance(network, WoodNet):
    print("Training full network")
    parameters = network.parameters()
else:
    print("Training only last layer of SqueezeNet")
    parameters = network.classifier[1].parameters()

optimizer = torch.optim.SGD(parameters, lr=0.001, momentum=0.9)
loss_function = nn.CrossEntropyLoss()

train_model(network, loss_function, optimizer, num_epochs=25)<jupyter_output>Training full network
Epoch 0/24
----------
Epoch: 0 (train): 100%|██████████| 6836/6836 [04:12<00:00, 27.05it/s]
train Loss: 0.5412 Acc: 0.7620
Epoch: 0 (val): 100%|██████████| 1465/1465 [00:51<00:00, 28.48it/s]
val Loss: 0.0882 Acc: 0.9692

Epoch 1/24
----------
Epoch: 1 (train): 100%|██████████| 6836/6836 [04:30<00:00, 25.27it/s]
train Loss: 0.0409 Acc: 0.9862
Epoch: 1 (val): 100%|██████████| 1465/1465 [00:53<00:00, 27.41it/s]
val Loss: 0.0184 Acc: 0.9937

Epoch 2/24
----------
Epoch: 2 (train): 100%|██████████| 6836/6836 [04:26<00:00, 25.61it/s]
train Loss: 0.0124 Acc: 0.9961
Epoch: 2 (val): 100%|██████████| 1465/1465 [00:50<00:00, 28.98it/s]
val Loss: 0.0147 Acc: 0.9950

Epoch 3/24
----------
Epoch: 3 (train): 100%|██████████| 6836/6836 [04:18<00:00, 26.45it/s]
train Loss: 0.0060 Acc: 0.9981
Epoch: 3 (val): 100%|██████████| 1465/1465 [00:51<00:00, 28.18it/s]
val Loss: 0.0084 Acc: 0.9975

Epoch 4/24
----------
Epoch: 4 (train):  17%|█▋        | 1140/6836 [00:43<03:43, 25.53it/s]<jupyter_text>Upload model weights and training logs to storage<jupyter_code># Upload checkpoints to storage
client = storage.Client.from_service_account_json("/content/drive/My Drive/## Project/TDT4173 Deep Learning Project-91d3b469375c.json")
bucket = client.get_bucket("tdt4173-datasets")

blob = bucket.blob("checkpoints/SqueezeNet-1605290736.1277423.data")
filename = "/content/checkpoints/SqueezeNet-1605290736.1277423.data"
blob.upload_from_filename(filename)

blob = bucket.blob("checkpoints/WoodNet-1605294933.5362356.data")
filename = "/content/checkpoints/WoodNet-1605294933.5362356.data"
blob.upload_from_filename(filename)
# Upload logs to storage
blob = bucket.blob("logs/SqueezeNet-1605290215.097698.csv")
filename = "/content/logs/SqueezeNet-1605290215.097698.csv"
blob.upload_from_filename(filename)

blob = bucket.blob("logs/SqueezeNet-1605290736.1277423.csv")
filename = "/content/logs/SqueezeNet-1605290736.1277423.csv"
blob.upload_from_filename(filename)

blob = bucket.blob("logs/WoodNet-1605294933.5362356.csv")
filename = "/content/logs/WoodNet-1605294933.5362356.csv"
blob.upload_from_filename(filename)<jupyter_output><empty_output><jupyter_text>Visualize the model performance for some images<jupyter_code>def visualize_model(model, num_images=6):
    was_training = model.training
    model.eval()
    images_so_far = 0
    fig = plt.figure()

    with torch.no_grad():
        for i, (inputs, labels) in enumerate(dataloaders['test']):
            inputs = inputs.to(device)
            labels = labels.to(device)

            outputs = model(inputs)
            _, preds = torch.max(outputs, 1)

            for j in range(inputs.size()[0]):
                images_so_far += 1
                ax = plt.subplot(num_images//2, 2, images_so_far)
                ax.axis('off')
                ax.set_title('predicted: {}'.format(class_names[preds[j]]))
                imshow(inputs.cpu().data[j])

                if images_so_far == num_images:
                    model.train(mode=was_training)
                    return
        model.train(mode=was_training)
visualize_model(squeezenet)

import cv2
inputs = [
    cv2.imread("/content/lars_1.png", cv2.IMREAD_COLOR),
    cv2.imread("/content/morgan_1.png", cv2.IMREAD_COLOR),
    cv2.imread("/content/morgan_2.png", cv2.IMREAD_COLOR),
    cv2.imread("/content/morgan_3.png", cv2.IMREAD_COLOR),
    cv2.imread("/content/ingvar_1.png", cv2.IMREAD_COLOR),
    cv2.imread("/content/dwayne_1.png", cv2.IMREAD_COLOR),
    cv2.imread("/content/kjartan_2.png", cv2.IMREAD_COLOR),
    cv2.imread("/content/faces/sampled_dataset_balanced_244/test/Kjartan/kjartan_video_5_9_augmentation_8.jpg", cv2.IMREAD_COLOR),
]

for i, inp in enumerate(inputs):
    inputs[i] = cv2.cvtColor(cv2.resize(inp, (244, 244)), cv2.COLOR_BGR2RGB)

def get_prediction_image(img, true_lab=None, plot=False):
    assert not plot or (plot and true_lab)
    mean = np.array([0.485, 0.456, 0.406])
    std = np.array([0.229, 0.224, 0.225])

    inp = cv2.resize(img, (224, 224)) / 255.0
    inp = inp / std - mean
    inp = inp.transpose((2, 0, 1))

    imgt = torch.Tensor(inp).unsqueeze(0).to(device)

    out = squeezenet(imgt)

    probabilities = F.softmax(out, dim=1)

    prob, class_idx = torch.max(probabilities, dim=1)
    pred = class_names[class_idx]

    if plot:
        plt.imshow(img)
        plt.text(5, 17,   f"Actual   : {true_lab}", color="white", fontsize=14)
        plt.text(5, 34,   f"Predicted: {pred}", color="white", fontsize=14)

    return pred, round(prob.item() * 100, 2), probabilities

get_prediction_image(inputs[2], "Morgan", plot=True)
plt.imshow(np.concatenate((inputs[1], inputs[2]), axis=1))
(pred, prob), actual = get_prediction_image(inputs[1]), "Morgan"
plt.text(5, 17,   f"Actual   : {actual}", color="white", fontsize=14)
plt.text(5, 34,   f"Predicted: {pred}", color="white", fontsize=14)
plt.text(5, 52,   f"[Certainty ({prob}%)]", color="white", fontsize=12)


(pred, prob), actual = get_prediction_image(inputs[2]), "Morgan"
plt.text(249, 17, f"Actual   : {actual}", color="white", fontsize=14)
plt.text(249, 34, f"Predicted: {pred}", color="white", fontsize=14)
plt.text(249, 52,   f"[Certainty ({prob}%)]", color="white", fontsize=12)

plt.savefig("morgan_crop_plot.png")
plt.show();<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup