<jupyter_start><jupyter_text>Perceptron Activation Network With a learning example and comparison_Overview:_ This is an implementation of the Perceptron Activation NetworkIn this notebook I present a model based on the biological brain, that combines Dropout, ReLU and Perceptron learning into a single learning algorithm - dubbed the Perceptron Activation Layer.  It is used like a typical Linear layer, however internally it implements a teacher-student signal for interrupting the flow of weights that do not contribute to the learning task.  A teacher network is responsible for computing a "teaching signal" that informs the "student network" which weights to activate and which to turn off.<jupyter_code># Import libraries
import matplotlib.pyplot as plt
# Visualizations will be shown in the notebook.
%matplotlib inline

import random
import os
import glob
import pickle
import time
import string
from tqdm import tqdm
from PIL import Image
import numpy as np
import json

import torch as torch
import torch.nn as nn    
import torch.nn.functional as F
from torch.nn.parameter import Parameter
from torch import optim
from torch.autograd import Variable
from torchvision import datasets, transforms, models
print("Using PyTorch version: " + str(torch.__version__))
use_gpu = torch.cuda.is_available()
torch.manual_seed(7)

print("Using GPU: {}".format(use_gpu))<jupyter_output>Using GPU: True<jupyter_text>**Setup the dataset**<jupyter_code>batch_size = 64
# Define a transform to normalize the data
transform = transforms.Compose([transforms.ToTensor(),
                                transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
                              ])
# Download and load the training data
trainset = datasets.MNIST('~/.pytorch/MNIST_data/train/', download=True, train=True, transform=transform)
testset = datasets.MNIST('~/.pytorch/MNIST_data/valid/', download=True, train=False, transform=transform)
train_loader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=False)
test_loader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False)<jupyter_output><empty_output><jupyter_text>** Utility functions **<jupyter_code># Accuracy and loss meter
class AverageMeter(object):
    """
    Computes and stores the average and
    current value.
    """
    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

# Visualise a layer activation
def viz_layer(layer):
    fig = plt.figure(figsize=(16, 12))
    n_filters = layer.size()[1]
    for i in range(n_filters):
        ax = fig.add_subplot(1, n_filters, i+1, xticks=[], yticks=[])
        # grab layer outputs
        ax.imshow(np.squeeze(layer[0,i].data.numpy()), cmap='gray')
        ax.set_title('Output %s' % str(i+1))
    
    plt.show()<jupyter_output><empty_output><jupyter_text>Build the networks **Build the LeNet5 network**<jupyter_code>class LeNet5(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(1, 6, 5, padding=2)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(16*5*5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)
        self.softmax = nn.LogSoftmax()
        
        
    def forward(self, x):
        # store last sample for visualisation
        self.lastx = x
        self.xc1 = self.conv1(x)
        self.xa1 = F.relu(self.xc1)
        x = F.max_pool2d(self.xa1, (2, 2))
        
        self.xc2 = self.conv2(x)
        self.xa2 = F.relu(self.xc2)
        x = F.max_pool2d(self.xa2, (2, 2))
        
        x = x.view(-1, self.num_flat_features(x))
        
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        x = self.softmax(x)
        
        return x 
    
    def num_flat_features(self, x):
        size = x.size()[1:]
        num_features = 1
        for s in size:
            num_features *= s
        return num_features<jupyter_output><empty_output><jupyter_text>**Define the Perceptron module**<jupyter_code>class Perceptron(nn.Module):
    def __init__(self, inputs, outputs, minimum = 0.0):
        super(Perceptron, self).__init__()
        self.inputs = inputs
        self.outputs = outputs
        
        # student network
        self.Sb = Parameter(torch.randn(self.outputs, requires_grad=True))
        self.SW = Parameter(torch.randn(self.outputs, self.inputs, requires_grad=True))
        
        # teacher network
        self.S_o, self.S_i = self.SW.size()
        self.Tb = Parameter(torch.randn(self.S_o * self.S_i, requires_grad=True))
        self.TW = Parameter(torch.randn(self.S_o * self.S_i, self.inputs, requires_grad=True))
        
        self.min_val = torch.tensor(minimum).float().cuda()
    
    def forward(self, x):
        # teacher signal
        self.Tz = F.linear(x, self.TW, self.Tb)
        self.Th = torch.matmul(self.Tz.t(), torch.ones(x.size()[0]).cuda())
        # perceptron gate
        self.o = torch.gt(self.Th, self.min_val).float()
        
        self.o = self.o.view(self.S_o, self.S_i)
        self.W = torch.mul(self.o, self.SW)
        
        # student signal
        x = F.linear(x, self.W, self.Sb)
        
        return x<jupyter_output><empty_output><jupyter_text>**Build the Perceptron Activation Network**<jupyter_code>class Tron(nn.Module):
    def __init__(self):
        super(Tron, self).__init__()
        self.conv1 = nn.Conv2d(1, 6, 5, padding=2)
        self.conv2 = nn.Conv2d(6, 16, 5)
        
        self.tron1 = Perceptron(16*5*5, 120)
        self.tron2 = Perceptron(120, 84)
        
        self.fc3 = nn.Linear(84, 10)
        self.softmax = nn.LogSoftmax()
        
    def forward(self, x):
        self.xc1 = self.conv1(x)
        self.xa1 = F.relu(self.xc1)
        x = F.max_pool2d(self.xa1, (2, 2))
        
        self.xc2 = self.conv2(x)
        self.xa2 = F.relu(self.xc2)
        x = F.max_pool2d(self.xa2, (2, 2))
        
        x = x.view(-1, self.num_flat_features(x))
        
        x = self.tron1(x)
        x = self.tron2(x)
        
        x = self.fc3(x)
        x = self.softmax(x)
        
        return x
    
    def num_flat_features(self, x):
        size = x.size()[1:]
        num_features = 1
        for s in size:
            num_features *= s
        return num_features<jupyter_output><empty_output><jupyter_text>Training the network **Loss plotting functions**<jupyter_code>def plot_training(train_losses, valid_losses, title):
    plt.subplot(1, 2, 1)
    plt.plot(train_losses, label='Training ' + title)
    plt.plot(valid_losses, label='Validation ' + title)
    plt.xlabel('epochs')
    plt.legend(frameon=False)
    plt.title(title)
    plt.show()<jupyter_output><empty_output><jupyter_text>**Train the network**<jupyter_code>def train(model, epoch, criterion, optimizer):
    
    model.train()
    train_loss = 0
    correct = 0
    num_train = len(train_loader.dataset)
    
    batch_time = AverageMeter()
    losses = AverageMeter()
    accs = AverageMeter()

    tic = time.time()
    with tqdm(total=num_train) as pbar:
        for batch_idx, (data, target) in enumerate(train_loader):
            if use_gpu:
                data, target = data.cuda(), target.cuda()

            data, target = Variable(data), Variable(target)

            optimizer.zero_grad()

            output = model(data)

            loss = criterion(output, target)
            loss.backward()

            train_loss += loss.item()

            predicted = torch.max(output, 1)[1]
            true_count = (predicted == target).float().sum()

            optimizer.step()
            
            correct += true_count
            acc = 100 * (true_count / len(target))

            # store
            losses.update(loss.item(), data.size()[0])
            accs.update(acc.item(), data.size()[0])
            
            # measure elapsed time
            toc = time.time()
            batch_time.update(toc-tic)

            pbar.set_description(
                (
                    "{:.1f}s - loss: {:.3f} - acc: {:.3f}".format(
                        (toc-tic), loss.item(), acc.item()
                    )
                )
            )
            pbar.update(batch_size)
        
    print('\nTrain set: Average loss: {:.4f}, Accuracy: {}/{} ({:.2f}%)'.format(
        train_loss, correct, len(train_loader.dataset),
        100. * correct / len(train_loader.dataset)))
        
    return train_loss, correct / len(train_loader.dataset)
def test(model, criterion, optimizer):
    model.eval()
    
    test_loss = 0
    correct = 0
    
    for data, target in test_loader:
        if use_gpu:
            data, target = data.cuda(), target.cuda()
        
        data, target = Variable(data), Variable(target)
        
        output = model(data)
        loss = criterion(output, target)
        
        test_loss += loss.item()
        predicted = torch.max(output, 1)[1]
        correct += (predicted == target).float().sum()
    
    print('Test set: Average loss: {:.4f}, Accuracy: {}/{} ({:.2f}%)\n'.format(
        test_loss, correct, len(test_loader.dataset),
        100. * correct / len(test_loader.dataset)))
    
    return test_loss, correct / len(test_loader.dataset)
# Train Model
def train_model(model, epochs):
    if (use_gpu):
        model.cuda()
    
    criteria = nn.NLLLoss()
    optimise = optim.SGD(model.parameters(), lr=0.003)
    scheduler = optim.lr_scheduler.StepLR(optimise, step_size=10, gamma=0.5)
    
    train_losses, train_accs = [], []
    valid_losses, valid_accs = [], []

    for e in range(epochs):
        print("Epoch: {} ...".format(e))
        scheduler.step()
        
        t_loss, t_acc = train(model, e, criteria, optimise)
        v_loss, v_acc = test(model, criteria, optimise)

        train_losses.append(t_loss)
        valid_losses.append(v_loss)
        train_accs.append(t_acc)
        valid_accs.append(v_acc)
    
    plot_training(train_losses, valid_losses, 'Loss')
    plot_training(train_accs, valid_accs, 'Accuracy')
# Training Parameters
epochs = 50<jupyter_output><empty_output><jupyter_text>**Train LeNet5**<jupyter_code># Standard LeNet5
lenet5 = LeNet5()
train_model(lenet5, epochs)<jupyter_output>Epoch: 0 ...<jupyter_text>**Train MLP Activation Network**<jupyter_code># Model with MLP Activation
tron = Tron()
# Training Model 2
train_model(tron, epochs)<jupyter_output>Epoch: 0 ...<jupyter_text>Visualise the activations **Visualise the input image**<jupyter_code>plt.imshow(np.squeeze(lenet5.lastx.cpu()[0,0].data.numpy()), cmap='gray')<jupyter_output><empty_output><jupyter_text>**Visualise the LetNet layers**<jupyter_code>viz_layer(lenet5.xc1.cpu())
viz_layer(lenet5.xa1.cpu())
viz_layer(lenet5.xc2.cpu())
viz_layer(lenet5.xa2.cpu())<jupyter_output><empty_output><jupyter_text>**Visualise the PAN layers**<jupyter_code>viz_layer(tron.xc1.cpu())
viz_layer(tron.xa1.cpu())
viz_layer(tron.xc2.cpu())
viz_layer(tron.xa2.cpu())<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup