# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python [conda env:deepfake]
#     language: python
#     name: conda-env-deepfake-py
# ---

from dfdc.core.core import *
from dfdc.core.video_core import *

from fastai.vision import *
from tqdm import tqdm

data_path = Path("/home/ubuntu/data/dfdc/")

detections_path = data_path/'dfdc_face_detections'

len(detections_path.ls()), detections_path.ls()[0]

croppped_path = data_path/'dfdc_cropped_faces'

len(croppped_path.ls()), croppped_path.ls()[0]

# ### 1) Read detections df and crop files

from dfdc.core.core import *

part_no = '4'

detections_df = read_face_detection_df(detections_path/f'part_{part_no}_retina_detections.csv')

detections_df.head()

detections_df['face_detections'][0]

detections_df.shape

for dets in detections_df['face_detections']: 
    if [o['detections'] for o in dets] == []: break

cropped_part_path = croppped_path/f'dfdc_train_part_{part_no}'

empty_dirs = [o for o in cropped_part_path.ls() if (o.ls()) == []] # no detections - false positives

cropped_files = get_files(cropped_part_path, extensions=['.jpg'], recurse=True)

len(cropped_files)

cropped_files[:4]

# ### 2) Collect and sort frames for each video

videoframes_dict = defaultdict(list)

for o in cropped_files: videoframes_dict[o.parent.name].append("/".join(str(o).split("/")[-3:]))

len(videoframes_dict)

for k,v in videoframes_dict.items(): 
    new_v = sorted(v, key=lambda o: int(Path(o).name.split("_")[1]))
    videoframes_dict[k] = new_v

res = []
for k,v in videoframes_dict.items(): res.append({"fname":f"{k}.mp4", "face_crop_fnames":v})
face_crop_fnames_df = pd.DataFrame(res)

face_crop_fnames_df.head()

source2fakes = defaultdict(list)
for _, row in detections_df.iterrows():
    if row['source'] != row['fname']: source2fakes[row['source']].append(row['fname'])

print(f"Total # of original videos: {len(source2fakes)}")


# ### 3) Compare original and fake frame pixels
#
# It is discussed in forums that there is at least 1 pixel change in every video

def plot_itemlist(il):
    n = int(np.ceil(np.sqrt(len(il))))
    axes = subplots(n,n).flatten()
    for i, ax in zip(range(len(il)), axes): il[i].show(ax=ax)


# +
# identical_idxs_dict = {}
# for source, fakes in source2fakes.items(): 
#     # get original frames itemlist
#     video_name = Path(source).stem
#     frame_fnames = videoframes_dict[video_name]
#     original_il = ImageList([croppped_path/o for o in frame_fnames])

#     # detect if there are unchanged fake frame  
#     identical_idxs = []
#     for i in range(len(fakes)):
#         video_name = Path(fakes[i]).stem
#         frame_fnames = videoframes_dict[video_name]
#         fake_il = ImageList([croppped_path/o for o in frame_fnames])
#         identical_idxs.append((np.where([torch.equal(orig_frame.data, fake_frame.data) 
#                            for orig_frame, fake_frame in zip(original_il, fake_il)])))
    
#     # 
#     identical_idxs_dict[source] = identical_idxs

# +
# plot_itemlist(original_il)
# plot_itemlist(fake_il)
# -

part_no = '0'

os.makedirs(data_path/'dfdc_training_csv', exist_ok=True)


def create_training_df(part_no):
    # get cropped files
    detections_df = pd.read_csv(detections_path/f'part_{part_no}_retina_detections.csv')
    cropped_files = get_files(croppped_path/f'dfdc_train_part_{part_no}', extensions=['.jpg'], recurse=True)

    # collect video frames relative to parent cropped faces dir
    videoframes_dict = defaultdict(list)
    for o in cropped_files: videoframes_dict[o.parent.name].append("/".join(str(o).split("/")[-3:]))

    # sort by frame #
    for k,v in videoframes_dict.items(): 
        new_v = sorted(v, key=lambda o: int(Path(o).name.split("_")[1]))
        videoframes_dict[k] = new_v

    # create dataframe
    res = []
    for k,v in videoframes_dict.items(): res.append({"fname":f"{k}.mp4", "face_crop_fnames":v})
    face_crop_fnames_df = pd.DataFrame(res)

    train_df = detections_df.merge(face_crop_fnames_df, on='fname', how='inner')
    train_df.to_csv(data_path/f'dfdc_training_csv/part_{part_no}_training.csv', index=False)
    return train_df


parts = list(map(str, range(50)))

for part in parts: create_training_df(part)

train_df











StarcoderdataJupyterScriptsDedupFiltered