<jupyter_start><jupyter_text>How to Train Your Own Cone Detection NetworksIn this notebook, we will demonstrate - how to train your own YOLOv3-based traffic cone detection network and do inference on a video.**[Accurate Low Latency Visual Perception for Autonomous Racing: Challenges Mechanisms and Practical Solutions](https://github.com/mit-han-lab/once-for-all)** is an accurate low latency visual perception system introduced by <NAME>, <NAME>, <NAME>, and <NAME>.  1. PreparationLet's first install all the required packages:<jupyter_code>! sudo apt install unzip
print('Installing PyTorch...')
! pip3 install torch 
print('Installing torchvision...')
! pip3 install torchvision 
print('Installing numpy...')
! pip3 install numpy 
# tqdm is a package for displaying a progress bar.
print('Installing tqdm (progress bar) ...')
! pip3 install tqdm 
print('Installing matplotlib...')
! pip3 install matplotlib 
print('Installing Tensorboard')
! pip3 install tensorboardx
print('Installing all the other required packages once for all')
! sudo python3 setup.py install
print('Installing video editor')
! sudo apt install ffmpeg -y<jupyter_output><empty_output><jupyter_text>Let' s clone our repo first...<jupyter_code>! git clone https://github.com/cv-core/MIT-Driverless-CV-TrainingInfra.git

! mv MIT-Driverless-CV-TrainingInfra/CVC-YOLOv3/* .<jupyter_output><empty_output><jupyter_text>Before we start training, let's download the Cone Detection dataset and the corresponding label and intial training weights.<jupyter_code>print("Downloading Training Dataset")
! wget https://storage.googleapis.com/mit-driverless-open-source/YOLO_Dataset.zip
! unzip -q YOLO_Dataset.zip
! mv YOLO_Dataset dataset/ && rm YOLO_Dataset.zip
print("Downloading YOLOv3 Sample Weights")
! wget https://storage.googleapis.com/mit-driverless-open-source/yolov3-training/sample-yolov3.weights 
print("Downloading Training and Validation Label")
! cd dataset/ && wget https://storage.googleapis.com/mit-driverless-open-source/yolov3-training/all.csv && cd ..
! cd dataset/ && wget https://storage.googleapis.com/mit-driverless-open-source/yolov3-training/train_mini_yolo.csv && mv train_mini_yolo.csv train.csv && cd ..
! cd dataset/ && wget https://storage.googleapis.com/mit-driverless-open-source/yolov3-training/validate_mini_yolo.csv && mv validate_mini_yolo.csv validate.csv && cd ..<jupyter_output><empty_output><jupyter_text>2. Using Pretrained YOLOv3 Weights File to Start Training First, import all the packages used in this tutorial:<jupyter_code>import os
import random
import tempfile
import time
import multiprocessing
import subprocess
import math
import shutil
import math

from datetime import datetime
import torch
import torch.nn as nn
from torch.utils.data import DataLoader

from models import Darknet
from utils.datasets import ImageLabelDataset
from utils.utils import model_info, print_args, Logger, visualize_and_save_to_local,xywh2xyxy
from yolo_tutorial_util import run_epoch
import validate
import warnings
import sys

##### section for all random seeds #####
torch.manual_seed(2)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
########################################

warnings.filterwarnings("ignore")
os.environ['CUDA_LAUNCH_BLOCKING'] = "1"

cuda = torch.cuda.is_available()
device = torch.device('cuda:0' if cuda else 'cpu')
num_cpu = multiprocessing.cpu_count() if cuda else 0

if cuda:
    torch.cuda.synchronize()
random.seed(0)
torch.manual_seed(0)

if cuda:
    torch.cuda.manual_seed(0)
    torch.cuda.manual_seed_all(0)
    torch.backends.cudnn.benchmark = True
    torch.cuda.empty_cache()<jupyter_output><empty_output><jupyter_text>Successfully imported all packages and configured random seed to 17! Training Config<jupyter_code># Training Related Config
batch_size = int(5)
optimizer_pick = "Adam"
model_cfg = "model_cfg/yolo_baseline.cfg"
weights_path = "sample-yolov3.weights"
output_path = "automatic"
dataset_path = "dataset/YOLO_Dataset/"
num_epochs = int(2) # Set them to 2048 during full dataset training
num_steps = 8388608
checkpoint_interval = int(1) # How often you want to get evaluation metric during training
val_tolerance = int(3)
min_epochs = int(3)

# Dataloader Related Config
data_aug = False # toggle for image augmentation
blur = False # Add blur to image
salt = False # Add "salt" noise to image
noise = False # Add noise to image
contrast = False # Add High Contrast to image
sharpen = False # Image Sharpen
ts = True # Tiling and Scaling
augment_affine = False # Affine
augment_hsv = False # HSV
lr_flip = False # left and right flip
ud_flip = False # up and down flip

# Training Hyperparameter Related Config
momentum = float(0.9)
gamma = float(0.95)
lr = float(0.001)
weight_decay = float(0.0)

xy_loss = float(2)
wh_loss= float(1.6)
no_object_loss = float(25)
object_loss = float(0.1)

# Debugging/Visualization Related Config
debug_mode = False
upload_dataset = False
vanilla_anchor = False
vis_batch = int(0)<jupyter_output><empty_output><jupyter_text>Initializing Model<jupyter_code>input_arguments = list(locals().items())

print("Initializing model")
model = Darknet(config_path=model_cfg,xy_loss=xy_loss,wh_loss=wh_loss,no_object_loss=no_object_loss,object_loss=object_loss,vanilla_anchor=vanilla_anchor)<jupyter_output><empty_output><jupyter_text>Processing Training Config<jupyter_code>if output_path == "automatic":
    current_month = datetime.now().strftime('%B').lower()
    current_year = str(datetime.now().year)
    if not os.path.exists(os.path.join('outputs/', current_month + '-' + current_year + '-experiments/' + model_cfg.split('.')[0].split('/')[-1])):
        os.makedirs(os.path.join('outputs/', current_month + '-' + current_year + '-experiments/' + model_cfg.split('.')[0].split('/')[-1]))
    output_uri = os.path.join('outputs/', current_month + '-' + current_year + '-experiments/' + model_cfg.split('.')[0].split('/')[-1])
else:
    output_uri = output_path

img_width, img_height = model.img_size()
bw  = model.get_bw()
validate_uri, train_uri = model.get_links()
num_validate_images, num_train_images = model.num_images()
conf_thresh, nms_thresh, iou_thresh = model.get_threshs()
num_classes = model.get_num_classes()
loss_constant = model.get_loss_constant()
conv_activation = model.get_conv_activation()
anchors = model.get_anchors()
onnx_name = model.get_onnx_name()

start_epoch = 0
weights_path = weights_path<jupyter_output><empty_output><jupyter_text>Data LoadersOne of our main contributions to vanilla YOLOv3 is the custom data loader we implemented:Each set of training images from a specific sensor/lens/perspective combination is uniformly rescaled such that their landmark size distributions matched that of the camera system on the vehicle. Each training image was then padded if too small or split up into multiple images if too large.<jupyter_code>with tempfile.TemporaryDirectory() as tensorboard_data_dir:
    print("Initializing data loaders")
    train_data_loader = torch.utils.data.DataLoader(
        ImageLabelDataset(train_uri, dataset_path=dataset_path, width=img_width, height=img_height, augment_hsv=augment_hsv,
                            augment_affine=augment_affine, num_images=num_train_images,
                            bw=bw, n_cpu=num_cpu, lr_flip=lr_flip, ud_flip=ud_flip,vis_batch=vis_batch,data_aug=data_aug,blur=blur,salt=salt,noise=noise,contrast=contrast,sharpen=sharpen,ts=ts,debug_mode=debug_mode, upload_dataset=upload_dataset),
        batch_size=(1 if debug_mode else batch_size),
        shuffle=(False if debug_mode else True),
        num_workers=(0 if vis_batch else num_cpu),
        pin_memory=cuda)
    print("Num train images: ", len(train_data_loader.dataset))

    validate_data_loader = torch.utils.data.DataLoader(
        ImageLabelDataset(validate_uri, dataset_path=dataset_path, width=img_width, height=img_height, augment_hsv=False,
                            augment_affine=False, num_images=num_validate_images,
                            bw=bw, n_cpu=num_cpu, lr_flip=False, ud_flip=False,vis_batch=vis_batch,data_aug=False,blur=False,salt=False,noise=False,contrast=False,sharpen=False,ts=ts,debug_mode=debug_mode, upload_dataset=upload_dataset),
        batch_size=(1 if debug_mode else batch_size),
        shuffle=False,
        num_workers=(0 if vis_batch else num_cpu),
        pin_memory=cuda)<jupyter_output><empty_output><jupyter_text>Initialize Optimizer<jupyter_code>if optimizer_pick == "Adam":
    print("Using Adam Optimizer")
    optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()),
                                lr=lr, weight_decay=weight_decay)
elif optimizer_pick == "SGD":
    print("Using SGD Optimizer")
    optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad, model.parameters()),
                            lr=lr, momentum=momentum, weight_decay=weight_decay)
else:
    raise Exception(f"Invalid optimizer name: {optimizer_pick}")
print("Loading weights")
model.load_weights(weights_path, model.get_start_weight_dim())

# Set scheduler
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=gamma)<jupyter_output><empty_output><jupyter_text>Sending Model to GPUs if we are in GPU mode  Let's Dance (Training)<jupyter_code>if torch.cuda.device_count() > 1:
    print('Using ', torch.cuda.device_count(), ' GPUs')
    model = nn.DataParallel(model)
model = model.to(device, non_blocking=True)

val_loss = 999  # using a high number for validation loss
val_loss_counter = 0
step = [0]  # wrapping in an array so it is mutable
epoch = start_epoch
while epoch < num_epochs and step[0] < num_steps:
    epoch += 1
    scheduler.step()
    model.train()
    run_epoch(label_prefix="train", data_loader=train_data_loader, epoch=epoch,
                step=step, model=model, num_epochs=num_epochs, num_steps=num_steps,
                optimizer=optimizer, device=device)
    print('Completed epoch: ', epoch)
    # Update best loss
    if epoch % checkpoint_interval == 0 or epoch == num_epochs or step[0] >= num_steps:
        # First, save the weights
        save_weights_uri = os.path.join(output_uri, "{epoch}.weights".format(epoch=epoch))
        model.save_weights(save_weights_uri)

        with torch.no_grad():
            print("Calculating loss on validate data")
            epoch_losses, epoch_time_total, epoch_num_targets = run_epoch(
                label_prefix="validate", data_loader=validate_data_loader, epoch=epoch,
                model=model, num_epochs=num_epochs, num_steps=num_steps, optimizer=None,
                step=step, device=device)
            avg_epoch_loss = epoch_losses[0] / epoch_num_targets
            print('Average Validation Loss: {0:10.6f}'.format(avg_epoch_loss))

            if avg_epoch_loss > val_loss and epoch > min_epochs:
                val_loss_counter += 1
                print(f"Validation loss did not decrease for {val_loss_counter}"
                        f" consecutive check(s)")
            else:
                print("Validation loss decreased. Yay!!")
                val_loss_counter = 0
                val_loss = avg_epoch_loss
                ##### updating best result for optuna study #####
                result = open("logs/result.txt", "w" )
                result.write(str(avg_epoch_loss))
                result.close() 
                ###########################################
            validate.validate(dataloader=validate_data_loader, model=model, device=device, step=step[0], bbox_all=False,debug_mode=debug_mode)
            if val_loss_counter == val_tolerance:
                print("Validation loss stopped decreasing over the last " + str(val_tolerance) + " checkpoints, creating onnx file")
                with tempfile.NamedTemporaryFile() as tmpfile:
                    model.save_weights(tmpfile.name)
                    weights_name = tmpfile.name
                    cfg_name = os.path.join(tempfile.gettempdir(), model_cfg.split('/')[-1].split('.')[0] + '.tmp')
                    onnx_gen = subprocess.call(['python3', 'yolo2onnx.py', '--cfg_name', cfg_name, '--weights_name', weights_name])
                    save_weights_uri = os.path.join(output_uri, onnx_name)
                    os.rename(weights_name, save_weights_uri)
                    try:
                        os.remove(onnx_name)
                        os.remove(cfg_name)
                    except:
                        pass
                break<jupyter_output><empty_output><jupyter_text>Our full dataset accuracy metrics for detecting traffic cones on the racing track:| mAP | Recall | Precision ||----|----|----|| 89.35% | 92.77% | 86.94% |  3. Inference Download target video file for inference<jupyter_code>from IPython.display import HTML
from base64 import b64encode

! wget https://storage.googleapis.com/mit-driverless-open-source/test_yolo_video.mp4

! ffmpeg -i test_yolo_video.mp4 test.mp4 && rm test_yolo_video.mp4

video_path = 'test.mp4'

mp4 = open(video_path,'rb').read()
decoded_vid = "data:video/mp4;base64," + b64encode(mp4).decode()
HTML(f'<video width=400 controls><source src={decoded_vid} type="video/mp4"></video>')<jupyter_output><empty_output><jupyter_text>Download pretrained weights for inference<jupyter_code>! wget https://storage.googleapis.com/mit-driverless-open-source/pretrained_yolo.weights<jupyter_output><empty_output><jupyter_text>Import all packages for inference<jupyter_code>import os
from os.path import isfile, join
import copy
import cv2
from tensorboardX import SummaryWriter
from PIL import Image, ImageDraw
import torchvision
from utils.nms import nms
from utils.utils import calculate_padding
from yolo_tutorial_util import single_img_detect, detect
from tqdm import tqdm
warnings.filterwarnings("ignore")
detection_tmp_path = "/tmp/detect/"<jupyter_output><empty_output><jupyter_text>Set up config file for inference<jupyter_code>target_path = "test.mp4"
output_path = "outputs/visualization/"
weights_path = "pretrained_yolo.weights"
conf_thres = float(0.8)
nms_thres = float(0.25)
cuda = torch.cuda.is_available()
device = torch.device('cuda:0' if cuda else 'cpu')
random.seed(0)
torch.manual_seed(0)
if cuda:
    torch.cuda.manual_seed(0)
    torch.cuda.manual_seed_all(0)
    torch.backends.cudnn.benchmark = True
    torch.cuda.empty_cache()
model = Darknet(config_path=model_cfg,xy_loss=xy_loss,wh_loss=wh_loss,no_object_loss=no_object_loss,object_loss=object_loss,vanilla_anchor=vanilla_anchor)

# Load weights
model.load_weights(weights_path, model.get_start_weight_dim())
model.to(device, non_blocking=True)

detect(target_path, output_path, model, device=device, conf_thres=conf_thres, nms_thres=nms_thres, detection_tmp_path=detection_tmp_path)
! cd outputs/visualization/ && ffmpeg -i test.mp4 output.mp4 && rm test.mp4 && cd ../..

video_path = "outputs/visualization/output.mp4"

mp4 = open(video_path,'rb').read()
decoded_vid = "data:video/mp4;base64," + b64encode(mp4).decode()
HTML(f'<video width=400 controls><source src={decoded_vid} type="video/mp4"></video>')<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup