<jupyter_start><jupyter_text>Unzipping Data<jupyter_code>### Unzipping Dataset

train_file = 'train'
test_file = 'test'
with zipfile.ZipFile("/CV_projects/catsDog_pytorch/train.zip","r") as train:
    train.extractall("")
    
with zipfile.ZipFile("/CV_projects/catsDog_pytorch/test.zip","r") as test:
    test.extractall("")
train_list = glob.glob(os.path.join(train_file,'*.jpg')) # Get list of all jpg files/dirs in data folder trian_dir
test_list = glob.glob(os.path.join(test_file, '*.jpg'))
print(f"Train Data: {len(train_list)}")
print(f"Test Data: {len(test_list)}")
print(os.listdir(train_file)[:5])<jupyter_output>['cat.0.jpg', 'cat.1.jpg', 'cat.10.jpg', 'cat.100.jpg', 'cat.1000.jpg']<jupyter_text>Data visualization<jupyter_code>from PIL import Image
random_idx = np.random.randint(1,25000,size=10)

fig = plt.figure()
i=1
for idx in random_idx:
    ax = fig.add_subplot(2,5,i)
    img = Image.open(train_list[idx])
    plt.imshow(img)
    i+=1

plt.axis('off')
plt.show()
device = (torch.device('cuda') if torch.cuda.is_available()
          else torch.device('cpu'))
print(f"Training on device : {device}.")
label = [path.split('\\')[-1].split('.')[0] for path in train_list]
print(type(label))
train_list[6071].split('\\')[-1].split('.')[0]<jupyter_output><empty_output><jupyter_text>Data split<jupyter_code>from sklearn.model_selection import train_test_split
train_list, val_list = train_test_split(train_list, test_size=0.2)
print(len(train_list))
print(len(val_list))<jupyter_output>20000
5000<jupyter_text>Data Augumentation<jupyter_code>train_transforms =  transforms.Compose([
        transforms.Resize((224, 224)),
        transforms.RandomCrop(224),
        transforms.RandomResizedCrop(224),
        transforms.RandomHorizontalFlip(p=0.7),
        transforms.ToTensor(),
    ])

val_transforms = transforms.Compose([
        transforms.Resize((224, 224)),
        transforms.RandomCrop(224),
        transforms.RandomResizedCrop(224),
        transforms.RandomHorizontalFlip(p=0.7),
        transforms.ToTensor(),
    ])


test_transforms = transforms.Compose([   
    transforms.Resize((224, 224)),
    transforms.RandomCrop(224),
     transforms.RandomResizedCrop(224),
    transforms.RandomHorizontalFlip(p=0.7),
    transforms.ToTensor()
    ])<jupyter_output><empty_output><jupyter_text>Datasets and Dataloaders<jupyter_code>class dataset(torch.utils.data.Dataset):
   
    def __init__(self,file_list,transform=None):
        self.file_list = file_list
        self.transform = transform
        
        
    #dataset length
    def __len__(self):
        self.filelength = len(self.file_list)
        return self.filelength
    
    #load images
    def __getitem__(self,idx):
        img_path = self.file_list[idx]
        img = Image.open(img_path)
        img_transformed = self.transform(img)
        
        label = img_path.split('\\')[-1].split('.')[0]
        
        if label == 'dog':
            label=1
        elif label == 'cat':
            label=0
            
        return img_transformed,label

train_data = dataset(train_list, transform=train_transforms)
test_data = dataset(test_list, transform=test_transforms)
val_data = dataset(val_list, transform=test_transforms)
trainloader = torch.utils.data.DataLoader(dataset = train_data, batch_size=8, shuffle=True )
testloader = torch.utils.data.DataLoader(dataset = test_data, batch_size=18, shuffle=False)
validloader = torch.utils.data.DataLoader(dataset = val_data, batch_size=8, shuffle=False)<jupyter_output><empty_output><jupyter_text>Data visualization<jupyter_code>for images, labels in trainloader:
    
    fig, ax = plt.subplots(figsize = (10, 10))
    ax.set_xticks([])
    ax.set_yticks([])
    ax.imshow(make_grid(images, 4).permute(1,2,0))
    break
print(len(validloader))
print(len(trainloader))
print(len(testloader))
img = next(iter(trainloader))[0][0]
plt.imshow(transforms.ToPILImage()(img))
print(f"Train data: {len(train_data)}, Train loader: {len(trainloader)}")

print(f"Val data: {len(val_data)}, Val loader: {len(validloader)}")
#check our images shape
train_data[0][0].shape<jupyter_output><empty_output><jupyter_text>TrainingLoop<jupyter_code>import datetime

def TrainingLoop (epoch, model, loss_fn, optimizer, trainloader ):
     for epoch in range(1, epoch+1):
         epoch_loss = 0
         train_correct = 0
         train_total = 0
         for data, label in trainloader:
            data = data.to(device)
            label = label.to(device)
        
            output = model(data)
            loss = loss_fn(output, label)
        
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
        
            epoch_loss += loss.item()
            _,tpredicted = torch.max(output, dim=1)
            train_total += label.shape[0]
            train_correct += int((tpredicted == label).sum())
         if epoch == 1 or epoch %10 == 0:
            print(f"Loss in epoch {epoch} : {epoch_loss/len(trainloader)} : date and time : {datetime.datetime.now()}: Got {train_correct} / {train_total} with train accuracy of :{float(train_correct) / float(train_total) * 100:.2f}")
    
    
         with torch.no_grad():
              valid_loss = 0
              correct = 0
              total = 0
              for data, label in validloader:
                  data = data.to(device=device)
                  label = label.to(device=device)
            ## Forward Pass
                  val_output = model(data)
                  val_loss = loss_fn(val_output, label)
                  valid_loss += val_loss.item()
                  _, predicted = torch.max(val_output, dim=1)
                  total += label.shape[0]
                  correct += int((predicted == label).sum())
              if epoch == 1 or epoch %10 == 0:
               print(
                   f"Loss in Valid epoch {epoch} : {valid_loss/len(validloader)} : Got {correct} / {total} with valid accuracy of :{float(correct) / float(total) * 100:.2f}")<jupyter_output><empty_output><jupyter_text>Transfer learning Resnet50<jupyter_code>from torchvision.models import resnet50
Net = resnet50(pretrained = True)

# Modifying Head of the classifier

Net.fc = nn.Sequential(
    nn.Dropout(0.3), #dropout incoming last convnet's few features
    nn.Linear(2048, 2)
)

#os.environ["CUDA_LAUNCH_BLOCKING"] = "1"
model =Net.to(device=device)
model.train()
optimizer = optim.SGD(params = model.parameters(), lr=1e-3)
loss_fn = nn.CrossEntropyLoss()
TrainingLoop(
    epoch=10,
    model=model,
    loss_fn=loss_fn,
    optimizer=optimizer,
    trainloader=trainloader,

)<jupyter_output>Loss in epoch 1 : 0.2243860801840201 : date and time : 2022-04-20 18:46:24.904942: Got 18168 / 20000 with train accuracy of :90.84
Loss in Valid epoch 1 : 0.16517693873271347 : Got 4679 / 5000 with valid accuracy of :93.58
Loss in epoch 10 : 0.08322408119685715 : date and time : 2022-04-20 19:32:00.444477: Got 19380 / 20000 with train accuracy of :96.90
Loss in Valid epoch 10 : 0.08550401275092735 : Got 4844 / 5000 with valid accuracy of :96.88<jupyter_text>Save and Load the model<jupyter_code>torch.save(model.state_dict(), "catsdog_resnet50.pt")
model = Net.to(device="cpu")
model.load_state_dict(torch.load("/CV_projects/catsDogs/catsdog_resnet50.pt"))
model.eval()<jupyter_output><empty_output><jupyter_text>Test data evaluation<jupyter_code>dog_probs = []
model.eval()
with torch.no_grad():
    for data, fileid in testloader:
        data = data.to(device="cpu")
        preds = model(data)
        preds_list = F.softmax(preds, dim=1)[:, 1].tolist()
        dog_probs += list(zip(list(fileid), preds_list))
    
dog_probs.sort(key = lambda x : int(x[0]))
dog_probs
idx = list(map(lambda x: x[0],dog_probs))
prob = list(map(lambda x: x[1],dog_probs))
submission = pd.DataFrame({'id':idx,'label':prob})
submission<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup