<jupyter_start><jupyter_text>MissOh DataLoader<jupyter_code># authored by haeyong.kang
# date : 2020/06/21<jupyter_output><empty_output><jupyter_text>AnotherMissOh Visual Structure- json_data['file_name'] : 'AnotherMissOh01.mp4'- json_data['visual_results']- json_data['visual_results'][0].keys() : dict_keys(['start_time', 'end_time', 'vid', 'image_info'])- {'start_time': '00:02:51;16', 'end_time': '00:02:54;15', 'vid': 'AnotherMissOh01_001_0078', 'image_info': ...}- json_data['visual_results'][0]['image_info']- [{'frame_id': 'AnotherMissOh01_001_0078_IMAGE_0000004295', 'place': 'none', 'persons': [{'person_id': 'Haeyoung1', 'person_info': {'face_rect': {'min_x': 515, 'min_y': 0, 'max_x': 845, 'max_y': 443}, 'full_rect': {'min_x': 278, 'min_y': 2, 'max_x': 1025, 'max_y': 769}, 'behavior': 'stand up', 'predicate': 'none', 'emotion': 'Neutral', 'face_rect_score': '0.5', 'full_rect_score': '0.9'}, 'related_objects': []}], 'objects': []}, - {'frame_id': 'AnotherMissOh01_001_0078_IMAGE_0000004311', 'place': '', 'persons': [{'person_id':'Haeyoung1','person_info': {'face_rect': {'min_x': 515, 'min_y': 0, 'max_x': 831, 'max_y': 411}, 'full_rect': {'min_x': 270, 'min_y': 0, 'max_x': 1025, 'max_y': 768}, 'behavior': 'stand up', 'predicate': 'none', 'emotion': 'Neutral', 'face_rect_score': '0.5', 'full_rect_score': '0.9'}, 'related_objects': []}],'objects': []},]<jupyter_code>import sys
sys.path.append("../") # go to parent dir
import os
from torch.utils.data import Dataset, DataLoader
import cv2
import pickle
import numpy as np
import glob
from torchvision.transforms import Compose, Resize, ToTensor, Normalize
from PIL import Image
import json
import argparse
import matplotlib.pyplot as plt

from Yolo_v2_pytorch.src.utils import *
from graphviz import Digraph, Graph

import networkx as nx
from networkx.drawing.nx_pydot import read_dot
#from networkx.drawing.nx_agraph import read_dot
from networkx.readwrite import json_graph
read_dot
def is_not_blank(s):
    return bool(s and s.strip())
MissOh_CLASSES = ['person']
print(MissOh_CLASSES[0])
global colors
colors = pickle.load(open("../Yolo_v2_pytorch/src/pallete", "rb"))
print(colors[0])
def get_args():
    parser = argparse.ArgumentParser(
        "You Only Look Once:Unified, Real-Time Object Detection")
    parser.add_argument("--image_size", type=int,
                        default=448,
                        help="The common width and height for all images")
    parser.add_argument("--batch_size", type=int, default=1,
                        help="The number of images per batch")
    # Training base Setting
    parser.add_argument("--momentum", type=float, default=0.9)
    parser.add_argument("--decay", type=float, default=0.0005)
    parser.add_argument("--dropout", type=float, default=0.5)
    parser.add_argument("--num_epoches", type=int, default=100)
    parser.add_argument("--test_interval", type=int, default=1,
                        help="Number of epoches between testing phases")
    parser.add_argument("--object_scale", type=float, default=1.0)
    parser.add_argument("--noobject_scale", type=float, default=0.5)
    parser.add_argument("--class_scale", type=float, default=1.0)
    parser.add_argument("--coord_scale", type=float, default=5.0)
    parser.add_argument("--reduction", type=int, default=32)
    parser.add_argument("--es_min_delta", type=float, default=0.0,
                        help="Early stopping's parameter:minimum change loss to qualify as an improvement")
    parser.add_argument("--es_patience", type=int, default=0,
                        help="Early stopping's parameter:number of epochs with no improvement after which training will be stopped. Set to 0 to disable this technique.")

    parser.add_argument("--pre_trained_model_type",
                        type=str, choices=["model", "params"],
                        default="model")
    parser.add_argument("--pre_trained_model_path", type=str,
                        default="Yolo_v2_pytorch/trained_models/only_params_trained_yolo_voc") # Pre-training path

    parser.add_argument("--saved_path", type=str,
                        default="./checkpoint") # saved training path
    parser.add_argument("--conf_threshold", type=float, default=0.35)
    parser.add_argument("--nms_threshold", type=float, default=0.5)
    args = parser.parse_args(args=[]) # for jupyter 
    return args
opt = get_args()
print(opt)
# visualize the images and labels
height, width = (768, 1024)
width_ratio = 448 / width
height_ratio = 448 / height
class AnotherMissOh(Dataset):
    def __init__(self, dataset, img_path, json_path, display_log=True):
        
        self.display_log = display_log
        self.init_clips(img_path)
        self.load_json(dataset,img_path, json_path)
        
    def init_clips(self, img_path):
        self.cnt_clips = 0
        self.img_path = img_path
        
        self.img_size = (1024, 768)
        self.img_scaled_size = (448, 448)
        
        tform = [
            Resize(self.img_scaled_size), # should match to Yolo_V2
            ToTensor(), 
            #Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) # should match to Yolo_V2
        ]
        self.transformations = Compose(tform)
        
        '''
        clips = {
            'episode' : [],
            'clip' : [],
            'start_time' : [],
            'end_time' : [],
            'vid' : [],
            'img_size' : [],
            'img_scaled_size' : [],
            'image_info' : []}
            
        image_info = {
            'frame_id': [],
            'place' : [],
            'persons' : []}
            
        persons = {
            'person_id': [],
            'face_rect' : [],
            'full_rect' : [],
            'behavior' : [],
            'predicate' : [],
            'emotion' : [],
            'face_rect_score' : [],
            'full_rect_score' : []}
        '''
        
    def load_json(self, dataset, img_path, json_path):
        
        self.clips = []
        
        for episode in dataset:
            img_dir = img_path + 'AnotherMissOh{:02}/'.format(episode)
            json_dir = json_path + 'AnotherMissOh{:02}_ver3.2.json'.format(episode)
            if self.display_log:
                print('imag_dir:{}'.format(img_dir))
                print('json_dir:{}'.format(json_dir))

            with open(json_dir, encoding='utf-8') as json_file:
                json_data = json.load(json_file)

            for i in range(len(json_data['visual_results'])):
                clip = {}
                clip['episode'] = []
                clip['clip'] = []
                clip['start_time'] = []
                clip['end_time'] = []
                clip['vid'] = []
                clip['image_info'] = []
                
                if self.display_log:
                    print("***{}th episode***{}th clips***************************************".format(episode, i))
                    print("['visual_results'][{}]['start_time']:{}".format(i,json_data['visual_results'][i]['start_time']))
                    print("['visual_results'][{}]['end_time']:{}".format(i,json_data['visual_results'][i]['end_time']))
                    print("['visual_results'][{}]['vid']:{}".format(i,json_data['visual_results'][i]['vid'].replace('_', '/')))
                    print("['visual_results'][{}]['img_size']:{}".format(i,img_size))
                    print("['visual_results'][{}]['img_scaled_size']:{}".format(i,img_scaled_size))
                    print("['visual_results'][{}]['episode']:{}".format(i,episode))
                
                clip['episode'].append(episode)
                clip['clip'].append(i)
                clip['start_time'].append(json_data['visual_results'][i]['start_time'])
                clip['end_time'].append(json_data['visual_results'][i]['end_time'])
                clip['vid'].append(json_data['visual_results'][i]['vid'].replace('_', '/'))
                
                
                for j, info in enumerate(json_data['visual_results'][i]['image_info']):
                    image_info = {}
                    image_info['frame_id'] = []
                    image_info['place'] = []
                    image_info['objects'] = {}
                    image_info['persons'] = {}
                    
                    if self.display_log:
                        print("=============={}th frame========================================".format(j))
                        
                    img_file = img_dir + json_data['visual_results'][i]['vid'].replace('_', '/')[-8:] + '/'+ info['frame_id'][-16:] + '.jpg'
                    image_info['frame_id'].append(img_file)
                    image_info['place'].append(info['place'])
                    
                    image_info['objects']['object_id']=[]
                    image_info['objects']['object_rect']=[]
                    for k, obj in enumerate(info['objects']):
                        image_info['objects']['object_id'].append(obj['object_id'])
                        object_bbox = obj['object_rect']
                        if (object_bbox['min_y'] == "" 
                            or object_bbox['max_y'] == "" 
                            or object_bbox['min_x'] == "" 
                            or object_bbox['max_x'] == ""):
                            object_rect = []
                            continue
                        else:
                            object_rect = [object_bbox['min_x'], object_bbox['min_y'], 
                                           object_bbox['max_x'], object_bbox['max_y']]
                        image_info['objects']['object_rect'].append(object_rect)
                    
                    image_info['persons']['person_id']=[]
                    image_info['persons']['face_rect']=[]
                    image_info['persons']['full_rect']=[]
                    image_info['persons']['behavior']=[]
                    image_info['persons']['predicate']=[]
                    image_info['persons']['emotion']=[]
                    image_info['persons']['face_rect_score']=[]
                    image_info['persons']['full_rect_score']=[]
                    
                    image_info['persons']['related_object_id']=[]
                    image_info['persons']['related_object_rect']=[]
                    
                    for k, person in enumerate(info['persons']):
                        if self.display_log:
                            print("--------------------{}th person-----------------------------".format(k))
                            
                        image_info['persons']['person_id'].append(person['person_id'])
                        
                        #import pdb; pdb.set_trace()
                        for j, robj in enumerate(person['related_objects']):
                            image_info['persons']['related_object_id'].append(robj['related_object_id'])
                            robj_bbox = robj['related_object_rect']
                            if (robj_bbox['min_y'] == "" 
                                or robj_bbox['max_y'] == "" 
                                or robj_bbox['min_x'] == "" 
                                or robj_bbox['max_x'] == ""):
                                related_object_rect = []
                                continue
                            else:
                                related_object_rect = [robj_bbox['min_x'], robj_bbox['min_y'], 
                                                       robj_bbox['max_x'], robj_bbox['max_y']]
                            image_info['persons']['related_object_rect'].append(related_object_rect)
    
                        face_bbox = person['person_info']['face_rect']
                        if (face_bbox['min_y'] == "" 
                            or face_bbox['max_y'] == "" 
                            or face_bbox['min_x'] == "" 
                            or face_bbox['max_x'] == ""):
                            face_rect = []
                            continue
                        else:
                            face_rect = [face_bbox['min_x'], face_bbox['min_y'], face_bbox['max_x'], face_bbox['max_y']]
                        image_info['persons']['face_rect'].append(face_rect)
                        full_bbox = person['person_info']['full_rect']
                        if (full_bbox['min_y'] == "" 
                            or full_bbox['max_y'] == "" 
                            or full_bbox['min_x'] == "" 
                            or full_bbox['max_x'] == ""):
                            full_rect = []
                            continue
                        else:
                            full_rect = [full_bbox['min_x'], full_bbox['min_y'], full_bbox['max_x'], full_bbox['max_y']]
                        image_info['persons']['full_rect'].append(full_rect)
                        image_info['persons']['behavior'].append(person['person_info']['behavior'])
                        image_info['persons']['predicate'].append(person['person_info']['predicate'])
                        image_info['persons']['emotion'].append(person['person_info']['emotion'])
                        image_info['persons']['face_rect_score'].append(person['person_info']['face_rect_score'])
                        image_info['persons']['full_rect_score'].append(person['person_info']['full_rect_score'])
                        
                    clip['image_info'].append(image_info)
                self.clips.append(clip)
                    
    def __len__(self):
        return len(self.clips)
                    
    def __getitem__(self, item):
        info = self.clips[item]['image_info']
        episode = self.clips[item]['episode']
        clip = self.clips[item]['clip']
        start_time = self.clips[item]['start_time']
        end_time = self.clips[item]['start_time']
        
        images = []
        for it, frame in enumerate(info):
            img = Image.open(frame['frame_id'][0]).convert('RGB')
            img = self.transformations(img)
            images.append(img)
        
        return images, info, episode, clip, start_time, end_time
img_path = '../data/AnotherMissOh/AnotherMissOh_images/'
json_path = '../data/AnotherMissOh/AnotherMissOh_Visual_ver3.2/'

episode = 1
train = [episode]
train_set = AnotherMissOh(train, img_path, json_path, False)
def graph(episode, scene, frm,st,et,info, save_file, debug = False):
    import string
    strseq = string.ascii_uppercase
    
    # define  graph
    dot = Digraph('G',filename='{}.gv'.format(save_file),engine='fdp')
    dot.attr('graph', rotate = '0', dpi='600',rankdir='TB', size='10,8')
    dot.attr('node', height='0.1', fontsize='6')
    dot.attr('edge', fontsize='6')

    place = "{}".format(info['place'][0])
    sound = "{}".format('sound')
    
    if not is_not_blank(place):
        place = 'none'
    if not is_not_blank(sound):
        sound = 'none'
        
    num_of_persons = len(info['persons']['person_id'])
    num_of_objects = len(info['objects']['object_id'])
    
    frm_graph = 'episode_{}_scene_{}_frame_{}'.format(
        episode, scene, frm)
    
    #dot.node(frm_graph, style='filled', color='lightgrey')
    episode_node = "episode_{:02d}".format(episode)
    scene_node = "scene_{:03d}".format(scene)
    frame_node = "frame_{:04d}".format(frm)
    dot.node(episode_node, style='filled', color='lightgrey')
    dot.node(scene_node, style='filled', color='lightgrey')
    dot.node(frame_node, style='filled', color='lightgrey')
    
    dot.node(place, style='filled', color='lightblue')
    dot.node(sound, style='filled', color='lightblue')
    
    if is_not_blank(episode_node) and is_not_blank(scene_node):
        dot.edge(episode_node, scene_node)
    
    if is_not_blank(scene_node) and is_not_blank(frame_node):
        dot.edge(scene_node, frame_node)
        
    if is_not_blank(frame_node) and is_not_blank(place):
        dot.edge(frame_node, place)
    
    if is_not_blank(frame_node) and is_not_blank(sound):
        dot.edge(frame_node, sound)
    
    for p in range(num_of_objects):
        try: 
            object_id = info['objects']['object_id'][p]
        except:
            object_id = 'none'
            #continue
            
        if is_not_blank(object_id) and object_id is not 'person':
            dot.node(object_id, style='filled', color='gold')
        if is_not_blank(frame_node) and is_not_blank(object_id):
            dot.edge(frame_node, object_id)
        
    for p in range(num_of_persons):
        
        try:
            person_id = "{}".format(info['persons']['person_id'][p])
        except:
            person_id = 'none'
            #continue
        try:
            behavior = "{}".format(info['persons']['behavior'][p])
        except:
            person_id = 'none'
            #continue
        try:
            predicate = "{}".format(info['persons']['predicate'][p])
        except:
            person_id = 'none'
            #continue
        try:
            emotion = "{}".format(info['persons']['emotion'][p])
        except:
            person_id = 'none'
            #continue
        try:
            robj_id = "{}".format(info['persons']['related_object_id'][p])
        except:
            robj_id = ''
            #continue


        if is_not_blank(person_id):
            dot.node(person_id)
        if is_not_blank(behavior):
            dot.node(behavior, style='filled', color='green')
        #if is_not_blank(predicate):
        #    dot.node(predicate, style='filled', color='yellow')
        if is_not_blank(emotion):
            dot.node(emotion, style='filled', color='blue')

        if is_not_blank(frame_node) and is_not_blank(person_id):
            dot.edge(frame_node, person_id)
        if is_not_blank(person_id) and is_not_blank(behavior):
            dot.edge(person_id, behavior)
        if is_not_blank(person_id) and is_not_blank(predicate) and is_not_blank(robj_id):
            dot.edge(person_id, robj_id, label=predicate, color='red')
            #dot.edge(predicate, robj_id)
        if is_not_blank(person_id) and is_not_blank(emotion):
            dot.edge(person_id, emotion)
            
        
            
    # show in image
    dot.format = 'png'
    dot.render('{}.gv'.format(save_file), view=True)

    graph = cv2.imread('{}.gv.png'.format(save_file))
    graph = cv2.resize(graph, dsize=(0, 0), fx=600.0/graph.shape[0], fy=600.0/graph.shape[0])

    if debug:
        plt.figure(figsize=(8,8))
        plt.imshow(graph)
        plt.show()
save_dir = '../results/drama_graph/'
if not os.path.exists(save_dir):
    os.makedirs(save_dir)
print(len(train_set))
debug = False

for scene in range(len(train_set)):
    #if scene < 1033:
    #    continue
        
    images, info, episode, scene, st, et = train_set[scene]
    scene= scene[0]
    episode= episode[0]

    for frm in range(len(images)):
        image = images[frm].cpu().numpy()
        #print(image)
        imageInfo = cv2.cvtColor(np.transpose(image * 255,(1,2,0)), 
                                 cv2.COLOR_RGB2BGR)

        frm_name = "episode_{:02d}_scene_{:03d}_frame_{:04d}".format(episode, scene, frm)
        save_file = save_dir + frm_name
        print("episode:{}, scene:{}, frame:{} st:{}, et:{}".format(
            episode, scene, frm, st, et))

        place = info[frm]['place'][0]
        sound = 'sound'
        num_of_persons = len(info[frm]['persons']['person_id'])
        num_of_objects = len(info[frm]['objects']['object_id'])
        graph(episode, scene, frm, st, et ,info[frm], save_file, debug)

        # read dot graph
        #dot_graph = nx.nx_pydot.read_dot('{}.gv'.format(save_file))
        #s_graph = json.dumps(json_graph.node_link_data(dot_graph))
        for p in range(num_of_objects):
            try: 
                object_id = info[frm]['objects']['object_id'][p]
            except:
                object_id = 'none'
                continue
            try:
                object_rect= info[frm]['objects']['object_rect'][p]
            except:
                object_rect = 'none'
                continue


            print("object_id:{}".format(object_id))
            print("object_rect:{}".format(object_rect))

            xmin_ = int(max(object_rect[0] * width_ratio, 0))
            ymin_ = int(max(object_rect[1] * height_ratio, 0))
            xmax_ = int(min((object_rect[2]) * width_ratio, 448))
            ymax_ = int(min((object_rect[3]) * height_ratio, 448))
            cv2.rectangle(imageInfo, (xmin_, ymin_), (xmax_, ymax_), 
                          (0.0, 128.0, 255.0), 2)

            cv2.putText(imageInfo, object_id, (xmin_, ymin_), 
                        cv2.FONT_HERSHEY_PLAIN, 1.5, (255.0, 255.0, 255.0), 1)


        for p in range(num_of_persons):
            try: 
                person_id = info[frm]['persons']['person_id'][p]
            except:
                person_id = 'none'
                #continue
            try:
                face_rect= info[frm]['persons']['face_rect'][p]
            except:
                face_rect = 'none'
                #continue
            try: 
                full_rect = info[frm]['persons']['full_rect'][p]
            except:
                full_rect = 'none'
                #continue
            try: 
                behavior = info[frm]['persons']['behavior'][p]
            except:
                behavior = 'none'
                #continue
            try:
                predicate = info[frm]['persons']['predicate'][p]
            except:
                predicate = 'none'
                #continue
            try: 
                emotion = info[frm]['persons']['emotion'][p]
            except:
                emotion = 'none'
                #continue

            print("place:{}".format(place))
            print("num_of_persons:{}".format(num_of_persons))
            print("person_id:{}".format(person_id))
            print("face_rect:{}".format(face_rect))
            print("full_rect:{}".format(full_rect))
            print("behavior:{}".format(behavior))
            print("predicate:{}".format(predicate))
            print("emotion:{}".format(emotion))

            # face rect
            xmin = int(max(face_rect[0] * width_ratio, 0))
            ymin = int(max(face_rect[1] * height_ratio, 0))
            xmax = int(min((face_rect[2]) * width_ratio, 448))
            ymax = int(min((face_rect[3]) * height_ratio, 448))
            cv2.rectangle(imageInfo, (xmin, ymin), (xmax, ymax), colors[0], 2)

            # full rect
            xmin = int(max(full_rect[0] * width_ratio, 0))
            ymin = int(max(full_rect[1] * height_ratio, 0))
            xmax = int(min((full_rect[2]) * width_ratio, 448))
            ymax = int(min((full_rect[3]) * height_ratio, 448))
            cv2.rectangle(imageInfo, (xmin, ymin), (xmax, ymax), colors[2], 2)
            cv2.putText(imageInfo, person_id, (xmin, ymin), 
                        cv2.FONT_HERSHEY_PLAIN, 1.5, (255.0, 255.0, 255.0), 1)
            cv2.putText(imageInfo, emotion, (xmin, ymax+20), 
                        cv2.FONT_HERSHEY_PLAIN, 1, (0.0, 0.0, 255.0), 1)
            cv2.putText(imageInfo, behavior, (xmin, ymax+30), 
                        cv2.FONT_HERSHEY_PLAIN, 1, (0.0, 255.0, 0.0), 1)
            cv2.putText(imageInfo, place, (10, 20), 
                        cv2.FONT_HERSHEY_PLAIN, 1.5, (255.0, 255.0, 255.0), 1)
            cv2.putText(imageInfo, sound, (10, 40), 
                        cv2.FONT_HERSHEY_PLAIN, 1.5, (255.0, 255.0, 255.0), 1)

        imageInfo = cv2.resize(imageInfo,dsize=(1024, 768))
        cv2.imwrite("{}_frame.png".format(save_file), imageInfo)
        imageInfo = cv2.cvtColor(imageInfo, cv2.COLOR_BGR2RGB)

        if debug:
            plt.figure(figsize=(8,8))
            plt.imshow(np.uint8(imageInfo))
            plt.show()
{"directed": true, "multigraph": true, 
 "graph": {"name": "G", "node": {"fontsize": "7", "height": "0.1"}, 
           "edge": {"fontsize": "7"}}, 
 "nodes": [{"color": "lightgrey", "style": "filled", "id": "episode_1"}, 
           {"color": "lightgrey", "style": "filled", "id": "scene_10"}, 
           {"color": "lightgrey", "style": "filled", "id": "frame_0"}, 
           {"color": "lightblue", "style": "filled", "id": "kitchen"}, 
           {"color": "lightblue", "style": "filled", "id": "sound"}, 
           {"id": "Jeongsuk"}, 
           {"color": "green", "style": "filled", "id": "cook"}, 
           {"color": "blue", "style": "filled", "id": "Happiness"}, 
           {"id": "Deogi"}], 
 "links": [{"source": "Jeongsuk", "target": "cook", "key": 0}, 
           {"source": "Jeongsuk", "target": "Happiness", "key": 0}, 
           {"source": "Deogi", "target": "cook", "key": 0}, 
           {"source": "Deogi", "target": "Happiness", "key": 0}]
}<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup