<jupyter_start><jupyter_text>PyTorch Experiments Template<jupyter_code>import torch
import torchvision
import pandas as pd
import numpy as np
import random
import cv2
import json
import matplotlib.pyplot as plt
import sys
import os
sys.path.insert(0, os.path.abspath("phd/src"))
sys.path.insert(0, os.path.abspath("benatools/src"))
import albumentations as A
import matplotlib.pyplot
from benatools.torch.efficient_net import create_efn2
from benatools.torch.fitter import TorchFitter
from benatools.torch.loss import CategoricalCrossEntropy
from benatools.utils.tools import MultiStratifiedKFold
from sklearn.model_selection import KFold
from sklearn.metrics import accuracy_score
from scipy.special import softmax

from ads.labels import get_topics
import ads.dataset

# CONSTANTS
PLATFORM = 'KAGGLE'  # this could be 'COLAB' or 'LOCAL'
DEVICE = 'TPU'   # This could be 'GPU' or 'CPU'



SEED = 42
seed_everything(SEED)<jupyter_output><empty_output><jupyter_text>Initialization Seeding everything for experiment replicability<jupyter_code># Seed
def seed_everything(seed):
    random.seed(seed)
    os.environ['PYTHONHASHSEED'] = str(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.backends.cudnn.deterministic = True

seed_everything(42)<jupyter_output><empty_output><jupyter_text>Read Data There are normally some files linked to the dataset with metadata, contextual information, calendars, etc.<jupyter_code># Read files
# training_examples = pd.read_csv('training_examples.csv')<jupyter_output><empty_output><jupyter_text>Dataset<jupyter_code>class TrainDataset(torch.utils.data.Dataset):
    def __init__(self, df, root:str, transforms=None, label_smoothing=0.0, channel_first=True, scaling_method='norm'):
        self.df = df  # DataFrame containing 
        self.root = root  # root folder
        self.transforms = transforms  # transforms pipeline
        self.label_smoothing = label_smoothing  # label smoothing alpha
        self.channel_first = channel_first  # whether to 
        self.scaling_method = scaling_method  # 'norm' normalizes the data to imagenet. 'scale' scales the data to [0,1]
        
    def get_labels(self):
        return np.array(self.df.columns)

    def _read(self, name):
        path = self.root + name
        img = cv2.imread(path)
        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
        img = cv2.resize(img, (256,256))
        return img

    def _label_smoothing(self, labels):
        if self.label_smoothing > 0:
            labels *= (1-self.label_smoothing)
            labels += (self.label_smoothing / labels.shape[1])
        return labels

    def _scale(self, img):
        if self.scaling_method == 'norm':
            normalize = A.Normalize(
                mean=[0.485, 0.456, 0.406],
                std=[0.229, 0.224, 0.225],
            )
            img = normalize(image=img)['image']
        else:
            img = img/255.0
        return img

    def _adjust_channel(self, img):
        if self.channel_first:
            img = np.transpose(img, axes=(2,0,1))
        return img

    def _transform(self, img):
        if self.transforms:
            img = self.transforms(image=img)['image']
        return img

    def __len__(self):
        return len(self.df)

    def __getitem__(self, idx):
        # Get row
        row = self.df.iloc[idx]
        labels = row.values

        # Label smoothing
        labels = self._label_smoothing(labels)

        # Read image and reformat
        img = self._read(row.name)

        # Apply transforms
        img = self._transform(img)

        # Scale
        img = self._scale(img)

        # Adjust to C x H x W for pytorch
        img = self._adjust_channel(img)
        
        # Format data into a dict
        data = {'x': torch.from_numpy(img),
                'y': torch.from_numpy(labels.astype(np.float32))
               }

        return data

    
def get_transforms():
    """
    A Function that returns a transforms pipeline
    """
    transform = A.Compose([
        A.OneOf([
            A.RandomRotate90(),
            A.Flip(),
            A.Transpose()
        ], p=0.2),
        A.OneOf([
            A.IAAAdditiveGaussianNoise(),
            A.GaussNoise(),
        ], p=0.2),
        A.OneOf([
            A.MotionBlur(p=.2),
            A.MedianBlur(blur_limit=3, p=0.1),
            A.Blur(blur_limit=3, p=0.1),
        ], p=0.2),
        A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.2, rotate_limit=45, p=0.5),
        A.OneOf([
            A.OpticalDistortion(p=0.3),
            A.GridDistortion(p=.1),
            A.IAAPiecewiseAffine(p=0.3),
        ], p=0.2),
        A.OneOf([
            A.CLAHE(clip_limit=2),
            A.IAASharpen(),
            A.IAAEmboss(),
            A.RandomBrightnessContrast(),            
        ], p=0.3),
        A.HueSaturationValue(p=0.3),
        A.OneOf([
                 A.Cutout(num_holes=100, max_h_size=6, max_w_size=6, fill_value=255, p=0.4),
                 A.Cutout(num_holes=8, max_h_size=25, max_w_size=25, fill_value=0, p=0.4),
                 A.ChannelDropout(channel_drop_range=(1, 1), fill_value=0, p=0.4)
        ]),
        
    ])
    return transform

def get_dataloader(df, bs=8, shuffle=False, drop_last=False, do_aug=True):
    transforms = None
    if do_aug:
        transforms = get_transforms()
    ds = ads.dataset.ImageDataset(df, root=IMG_ROOT, transforms=transforms)
    return torch.utils.data.DataLoader(ds, batch_size=bs, shuffle=shuffle, num_workers=4, pin_memory=True, drop_last=drop_last)<jupyter_output><empty_output><jupyter_text>It is useful to take a look at the data<jupyter_code>bs = 12
dl = get_dataloader(df.iloc[:bs], bs=bs, shuffle=False, drop_last=False, do_aug=True)
fig, axis = plt.subplots(2,bs//2, figsize=(20,10))
axis = axis.ravel()
for data in dl:
    for i in range(len(data)):
        axis[i].set_title(' | '.join( df.columns[data[i]['y'].numpy()==1] ) )
        axis[i].imshow(np.transpose(data[i]['x'].numpy(), (1,2,0)))<jupyter_output><empty_output><jupyter_text>ModelWhen experimenting, many different models or variations can be tried.  It is useful to have a common function to route the model creations further in the training loop<jupyter_code>class Identity(torch.nn.Module):
    def __init__(self):
        super(Identity, self).__init__()
        
    def forward(self, x):
        return x

class ImageClassifier(torch.nn.Module):
    def __init__(self, n_outs=39, trainable_base=False):
        super(ImageClassifier, self).__init__()
        self.base = torchvision.models.resnet152(pretrained=True, progress=True)
        self.base.fc = Identity()
        
        self.set_trainable(trainable_base)

        self.classifier = torch.nn.Sequential(
          torch.nn.Linear(in_features=2048, out_features=512),
          torch.nn.ReLU(),
          torch.nn.LayerNorm(512),
          torch.nn.Dropout(0.25),
          torch.nn.Linear(in_features=512, out_features=n_outs),
        )

    def set_trainable(self, trainable):
        for param in self.base.parameters():
            param.requires_grad = trainable

    def get_cnn_outputs(self, b):
        outs = [1280, 1280, 1408, 1536, 1792, 2048, 2064, 2560]
        return outs[b]
        
    def forward(self, x):
        x = self.base(x)
        x = self.classifier(x)
        return x<jupyter_output><empty_output><jupyter_text>Experiments Configuration<jupyter_code>N_EXPERIMENTS = 1  # Normally not more than one run per commit
FOLDS = [0] * N_EXPERIMENTS # Each run should cover a single fold

# DATALOADER PARAMS
BS = [32] * N_EXPERIMENTS


# LEARNING RATE
LR = [0.001] * N_EXPERIMENTS

# TRANSFORMS
# Params for the transforms functions

# GLOBAL PARAMETERS
EPOCHS=50
DISPLAY_PLOT=True
VERBOSE = 1<jupyter_output><empty_output><jupyter_text>Training Loop<jupyter_code># Reduce data to a subsample
df_sub = df #.iloc[:10000]

cv = MultiStratifiedKFold(5, df_sub, df.columns.tolist(), seed=SEED)
cv_dict = {i:(train_idx, val_idx) for i,(train_idx, val_idx) in enumerate(cv.split(df_sub))}

for i in range(0, N_EXPERIMENTS):
    print(f'********** EXPERIMENT {i} **********')
    print(f'***** bs train {BS[i]} *****')
    print(f'***** LR {LR[i]} *****')
    print(f'**********************************\n')

    seed_everything(SEED)

    # Get Dataloader
    train_idx, val_idx = cv_dict[FOLDS[i]]
    train_df, val_df = df_sub.loc[train_idx], df_sub.loc[val_idx]
    print(f'Training on {len(train_df)} samples - Validating on {len(val_df)} samples')
    train_ds = get_dataloader(train_df, bs=BS[i], shuffle=True, drop_last=False, do_aug=True)
    val_ds = get_dataloader(val_df, bs=BS[i], shuffle=False, drop_last=False, do_aug=False)

    # Create model
    model = ImageClassifier(trainable_base=True)
    #optimizer = torch.optim.Adam(model.parameters(), lr=0.001 )
    optimizer = torch.optim.SGD(model.parameters(), lr=LR[i], momentum=0.9)
    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.1, patience=3, mode='max')
    
    #loss = torch.nn.BCEWithLogitsLoss()
    loss = CategoricalCrossEntropy(from_logits=True, label_smoothing=0.1, reduction='mean')
    model.cuda()

    # Fitter object
    fitter = TorchFitter(model, device='cuda', loss=loss, optimizer=optimizer, scheduler=scheduler )
    history = fitter.fit(train_ds, val_ds, n_epochs=EPOCHS, metric=accuracy_one_hot, early_stopping_mode='max', verbose_steps=5, early_stopping=10)

    # Plot training
    plt.figure(figsize=(15,5))
    plt.plot(np.arange(len(history)), history['train'],'-o',label='Train Loss',color='#ff7f0e')
    plt.plot(np.arange(len(history)), history['val'],'-o',label='Val Loss',color='#1f77b4')
    x = np.argmin( history['val'] ); y = np.min( history['val'] )
    xdist = plt.xlim()[1] - plt.xlim()[0]; ydist = plt.ylim()[1] - plt.ylim()[0]
    plt.text(x-0.03*xdist,y-0.13*ydist,'min loss\n%.2f'%y,size=14)
    plt.ylabel('Loss',size=14); plt.xlabel('Epoch',size=14)
    plt.legend(loc=2)
        
    plt2 = plt.gca().twinx()
    plt2.plot(np.arange(len(history)),history['val_metric'],'-o',label='Accuracy',color='#36de47')
    #x = np.argmax( history['val_F1'] ); y = np.max( history['val_F1'] )
    #xdist = plt2.xlim()[1] - plt2.xlim()[0]; ydist = plt2.ylim()[1] - plt2.ylim()[0]
    #plt2.text(x-0.03*xdist,y-0.13*ydist,'max F1\n%.2f'%y,size=14)
    #plt2.ylabel('F1',size=14); plt2.xlabel('Epoch',size=14)
    plt2.legend()
        
    #plt2 = plt.gca().twinx()
    #plt2.plot(np.arange(len(history)),history['lr'],'-o',label='LR',color='#2ca02c')
    #plt.ylabel('LR',size=14)
        
    plt.title(f'Results fold {i}',size=18)
    plt.legend(loc=3)
    plt.show()<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup