<jupyter_start><jupyter_text>EvaluationEvaluate model prediction for an entire play.<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import sys, os
sys.path.append('/home/ec2-user/SageMaker/helmet_detection/src')
from helmet_detection_model.detector import ObjectDetector
video_in = '/home/ec2-user/SageMaker/helmet_detection/input/train/57583_000082_Endzone.mp4'
model_path = '/home/ec2-user/SageMaker/helmet_detection/model/model_helmet_frcnn.pt'
gtfile_name = '/home/ec2-user/SageMaker/helmet_detection/input/train_labels.csv'
full_video = True 
subset_video = 4 
conf_thres=0.9
iou_threshold = 0.25
num_classes = 2
# %%time
# detections, eval_det, fns, fps = ObjectDetector.run_detection_eval_video(video_in, gtfile_name, 
#                                                                             model_path, full_video,
#                                                                             subset_video, conf_thres,
#                                                                             iou_threshold)
# eval_det.describe()<jupyter_output><empty_output><jupyter_text>Draw detection errors on frames<jupyter_code># eval_det.to_csv("/home/ec2-user/SageMaker/helmet_detection/output/eval_det.csv", index=False)
# fns.to_csv("/home/ec2-user/SageMaker/helmet_detection/output/fns.csv", index=False)
# fps.to_csv("/home/ec2-user/SageMaker/helmet_detection/output/fps.csv", index=False)
eval_det = pd.read_csv("/home/ec2-user/SageMaker/helmet_detection/output/eval_det.csv")
fns = pd.read_csv("/home/ec2-user/SageMaker/helmet_detection/output/fns.csv")
fps = pd.read_csv("/home/ec2-user/SageMaker/helmet_detection/output/fps.csv")

fn_thres = 3
fp_thres = 3
# # list of frames with fn>=fn_thres and fp>=fp_thres
frame_list = eval_det[(eval_det['fn'] >= fn_thres) & (eval_det['fp'] >= fp_thres)]['frame_id'].tolist()
## frame_list = ObjectDetector.find_frames_high_fn_fp(eval_det, fn_thres, fp_thres)

# # list of frames with no fn and fp
# frame_list = eval_det[(eval_det['fn'] == 0) & (eval_det['fp'] == 0)]['frame_id'].tolist()

# list of frames with more than 5 fn
# frame_list = eval_det[(eval_det['fn'] > 5)]['frame_id'].tolist()
print(frame_list)
fns.shape
!rm /home/ec2-user/SageMaker/helmet_detection/output/out_images/*
success = ObjectDetector.draw_detect_error(video_in, gtfile_name, full_video, subset_video, frame_list, fns, fps)
success<jupyter_output><empty_output><jupyter_text>Get %frame with no fn and fp, fn=1, fn between 2 and 5, and fn more than 5<jupyter_code>df_good = eval_det[(eval_det['fn'] == 0) & (eval_det['fp'] == 0)]
print(df_good.shape)
print(100*(df_good.shape[0]/eval_det.shape[0]))
df_fn_1 = eval_det[(eval_det['fn'] == 1)]
print(df_fn_1.shape)
print(100*(df_fn_1.shape[0]/eval_det.shape[0]))
df_fn_2_5 = eval_det[(eval_det['fn'] >= 2) & (eval_det['fn'] <= 5)]
print(df_fn_2_5.shape)
print(100*(df_fn_2_5.shape[0]/eval_det.shape[0]))
df_fn_5 = eval_det[(eval_det['fn'] > 5)]
print(df_fn_5.shape)
print(100*(df_fn_5.shape[0]/eval_det.shape[0]))
df_fn_5
eval_det["precision"] = eval_det.apply(lambda row: row.tp/(row.tp + row.fp), axis=1)
eval_det["recall"] = eval_det.apply(lambda row: row.tp/(row.tp + row.fn), axis=1)
eval_det["f1_score"] = eval_det.apply(lambda row: (2 * row.precision * row.recall)/(row.precision + row.recall), axis=1)
eval_det.head()
# Calculate total number of helmets, tp, fn, fp, precision, recall, and F1 score 
total_gt = eval_det['num_object_gt'].sum()
total_tp = eval_det['tp'].sum()
total_fn = eval_det['fn'].sum()
total_fp = eval_det['fp'].sum()
total_precision = total_tp/(total_tp+total_fp)
total_recall = total_tp/(total_tp+total_fn)
total_f1 = 2*total_precision*total_recall/(total_precision+total_recall)
total_gt, total_tp, total_fn, total_fp, total_precision, total_recall, total_f1<jupyter_output><empty_output><jupyter_text>Plot R/R and FN/FP for each frame<jupyter_code>snap_time=10 
fig, ax1 = plt.subplots()

ax1.set_xlabel('Frame ID')
ax1.set_ylabel('Precision / Recall', color='g')
ax1.plot(eval_det['frame_id'], eval_det['precision'], color='r')
ax1.plot(eval_det['frame_id'], eval_det['recall'], color='g')
ax1.set_ylim([0, 1])
plt.legend(['precision', 'recall'], loc='upper left')

ax2 = ax1.twinx()  # instantiate a second axes that shares the same x-axis

ax2.set_ylabel('FN / FP', color='b')  # we already handled the x-label with ax1
ax2.plot(eval_det['frame_id'], eval_det['fn'], color='c')
ax2.plot(eval_det['frame_id'], eval_det['fp'], color='b')
ax2.set_ylim([0, 22])
ax2.plot(eval_det['frame_id'], eval_det['num_object_gt'], color='k')

ax2.tick_params(axis='y', labelcolor='b')
plt.legend(['FN', 'FP', 'Total Helmets'], loc='lower right')
fig.tight_layout()  # otherwise the right y-label is slightly clipped
plt.axvline(x=snap_time, color='k', linestyle='--')
# plt.show()
plt.savefig('/home/ec2-user/SageMaker/0Artifact/helmet_detection/output/pr_fnfp.png')<jupyter_output><empty_output><jupyter_text>Plot F1 score and FN/FP for each frame<jupyter_code>fig, ax1 = plt.subplots()

ax1.set_xlabel('Frame ID')
ax1.set_ylabel('F1 score', color='g')
ax1.plot(eval_det['frame_id'], eval_det['f1_score'], color='r')
ax1.set_ylim([0, 1])
plt.legend(['F1 score'], loc='upper left')
# ax1.tick_params(axis='y', labelcolor=color)

ax2 = ax1.twinx()  # instantiate a second axes that shares the same x-axis

ax2.set_ylabel('FN / FP', color='b')  # we already handled the x-label with ax1
ax2.plot(eval_det['frame_id'], eval_det['fn'], color='c')
ax2.plot(eval_det['frame_id'], eval_det['fp'], color='b')
ax2.set_ylim([0, 22])
ax2.plot(eval_det['frame_id'], eval_det['num_object_gt'], color='k')

ax2.tick_params(axis='y', labelcolor='b')
plt.legend(['FN', 'FP', 'Total Helmets'], loc='lower right')
fig.tight_layout()  # otherwise the right y-label is slightly clipped
plt.axvline(x=snap_time, color='k', linestyle='--')# plt.show()
plt.savefig('/home/ec2-user/SageMaker/0Artifact/helmet_detection/output/f1_fnfp.png')<jupyter_output><empty_output><jupyter_text>Plot stacked bar for tp, fn and fp for each frame<jupyter_code># pal = ["#9b59b6", "#e74c3c", "#34495e", "#2ecc71"]
pal = ["g","r","b"]
plt.figure(figsize=(12,8))
plt.stackplot(eval_det['frame_id'], eval_det['tp'], eval_det['fn'], eval_det['fp'], 
              labels=['TP','FN','FP'], colors=pal)
plt.plot(eval_det['frame_id'], eval_det['num_object_gt'], color='k', linewidth=6, label='Total Helmets')
plt.legend(loc='best', fontsize=12)
plt.xlabel('Frame ID', fontsize=12)
plt.ylabel(' # of TPs, FNs, FPs', fontsize=12)
plt.axvline(x=snap_time, color='k', linestyle='--')
plt.savefig('/home/ec2-user/SageMaker/0Artifact/helmet_detection/output/stacked.png')

detections = ObjectDetector.run_detection_video(video_in, model_path, full_video,subset_video, conf_thres)
vid_title = "/home/ec2-user/SageMaker/helmet_detection/input/" + os.path.splitext(os.path.basename(video_in))[0] + '.csv'
print(vid_title)
detections.to_csv(vid_title, index=None)
!ls /home/ec2-user/SageMaker/0Artifact/helmet_detection/input/train_labels.csv
# !mkdir src/helmet_detection_metric/detections
# !mkdir src/helmet_detection_metric/groundtruths
# !mkdir src/helmet_detection_metric/results
!python src/helmet_detection_metric/object_detection_metrics.py '/home/ec2-user/SageMaker/0Artifact/helmet_detection/input/train/57583_000082_Endzone.mp4' True 0 4000
!python src/helmet_detection_metric/pascalvoc.py<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup