<jupyter_start><jupyter_text>The originals are not included in the clusters dataframe.<jupyter_code>cluster_ids, cluster_counts = np.unique(np.concatenate([clusters.cluster_1.dropna().values,
                                       clusters.cluster_2.dropna().values]), return_counts=True)
#sort
cluster_counts, cluster_ids = zip(*sorted(zip(cluster_counts.tolist(), cluster_ids.tolist()),reverse=True))
cluster_ids = list(cluster_ids)

print("Number of unique clusters {0}".format(len(cluster_ids)))
for i, (c, count) in enumerate(zip(cluster_ids, cluster_counts)):
    print("Cluster id {0}: {1} original videos".format(int(c),count))
    if i >= 10:
        break
## Merge data
meta_tmp = meta.sample(frac=1, random_state=SEED)
meta_tmp = meta_tmp.merge(clusters[['original','cluster_1','cluster_2', 'both_missing']].drop_duplicates(), 
                          on='original', how='left')<jupyter_output><empty_output><jupyter_text>Add clusters to the original videos. Clusters are the same in any of their fake replicates so just copy the clusters from the first one.<jupyter_code>for original_index in tqdm(meta_tmp[meta_tmp['original'].isna()].index.values):
    original_video = meta_tmp.loc[original_index,'index']
    
    # get fake replicates
    fakes = meta_tmp[meta_tmp['original'] == original_video]
    if len(fakes) > 0:
        cluster_1 = fakes.iloc[0]['cluster_1']
        cluster_2 = fakes.iloc[0]['cluster_2']
        both_missing = fakes.iloc[0]['both_missing']
        
        # set the same clusters to the original
        meta_tmp.at[original_index,'cluster_1'] = cluster_1
        meta_tmp.at[original_index,'cluster_2'] = cluster_2
        meta_tmp.at[original_index,'both_missing'] = both_missing
meta_tmp[meta_tmp['original'].isna()].head()
## Open fake types and merge columns
fake_types_df = pd.read_csv('../data/fake_type_classes.csv')
meta_tmp = meta_tmp.merge(fake_types_df[['index','first_person_label', 'second_person_label']], on='index')
meta_tmp.head()
## To make sure there is no leak between videos related to the original and same actors
## we generate a grouper column based on the original video and, in case former is missing, the actor cluster
## the video belongs to

## If cluster_1 exist choose it, else "Y"
meta_tmp['name_index'] = np.where(meta_tmp.cluster_1.notna(),
                                  meta_tmp.cluster_1.astype(str),
                                  "Y")
meta_tmp['name_index'] = meta_tmp.name_index.astype(str)
meta_tmp.head()
name_counts = meta_tmp.name_index.value_counts()
name_counts
# How many error cases?
name_counts['Y']
sns.distplot(name_counts.values, kde=False)
plt.xlabel('Number of videos in a cluster')
plt.ylabel('Number of clusters')<jupyter_output><empty_output><jupyter_text>A few of the clusters have over 1K videos but majority has less than 200. The smallest number of videos per cluster is two which means one real and one corresponding fake video.We want to split these clusters first, into train and validation sets. It would be ideal to have sn equal amount of each fake classes in the valdation part.<jupyter_code>cluster_info = pd.DataFrame({'cluster':name_counts.keys(), 
                             'count':list(name_counts.values),
                             'type-0':0,
                             'type-1':0,
                             'type-2':0,
                             'type-3':0,
                             'type-4':0})
for i in tqdm(cluster_info.index.values):
    c = cluster_info.iloc[i]['cluster']
    sample = meta_tmp[meta_tmp['name_index'] == c]
    for faketype in range(5):
        cluster_info.at[i,'type-{0}'.format(faketype)] = len(sample[sample['first_person_label'] == float(faketype)])

cluster_info.head()
def plotTypeDistribution(clusters=[], return_only_values=False):
    df = pd.DataFrame({'type':[0,1,2,3,4],'count':[0,0,0,0,0]})
    for cluster in clusters:
        sample = cluster_info[cluster_info['cluster']==cluster]
        for faketype in range(5):
            df.at[faketype,'count'] = df.at[faketype,'count'] + sample['type-{0}'.format(faketype)]
    # calculate percentage from the total set
    sample_count = 0
    for faketype in range(5):
        sample_count += df.at[faketype,'count']
    min_count = min(df['count'].values)
    max_count = max(df['count'].values)
    min_max_ratio = min_count/max_count
    percentage = sample_count/len(meta_tmp)
    if return_only_values:
        return percentage, min_max_ratio
    sns.barplot(x='type',y='count',data=df)
    plt.suptitle('{0} samples = {1:.2f} % from all. min_count/max_count: {2:.2f}'.format(sample_count, 
                                                                                         100*percentage,
                                                                                         min_max_ratio))
    plt.show()
    
# plot distribution from all
plotTypeDistribution(name_counts.keys())<jupyter_output><empty_output><jupyter_text>Type 2 is the most common and type 4 the least common with overr ten times the difference between these two classes.For validation set, sample from medium size clusters (40-200 videos) so that all classes are equally represented.<jupyter_code>from itertools import permutations
import random

min_val_ratio = 1.0 - TRAIN_PCT - 0.02
max_val_ratio = 1.0 - TRAIN_PCT + 0.02
print("Accept val set size from {0:.2f} to {1:.2f}".format(min_val_ratio,max_val_ratio))

highest_min_max_ratio = 0
validation_index_list = []
stopping_condition = 0.3

# brute force search for suitable combination
range_list = list(range(200,460))
random.shuffle(range_list)
for combination in tqdm(permutations(range_list, 160)):
    percentage, min_max_ratio = plotTypeDistribution(name_counts.keys()[list(combination)], return_only_values=True)
    if percentage > min_val_ratio and percentage < max_val_ratio:
        if min_max_ratio > highest_min_max_ratio:
            highest_min_max_ratio = min_max_ratio
            validation_index_list = list(combination)
            print("Found better ratio: {0:.3f}".format(highest_min_max_ratio))
            if highest_min_max_ratio >= stopping_condition:
                print("Stopping condition reached")
                break
plotTypeDistribution(name_counts.keys()[validation_index_list])<jupyter_output><empty_output><jupyter_text>This is good enough. The type classes are not necessary evenly spaced and this may explain why class 2 is the most common.<jupyter_code># Validation clusters
assert "Y" not in list(name_counts.keys()[validation_index_list])
_ = [print(name_counts.keys()[ind]) for ind in validation_index_list]
meta_tmp['isVal'] = np.where(meta_tmp.cluster_1.isin(list(name_counts.keys()[validation_index_list])),
                             True,
                             False)
meta_tmp.head()
## Split the grouper into train and val using numpy's array_split
tmp_df = pd.DataFrame()
name_df = meta_tmp[~meta_tmp['name_index'].isin(["Y"])][['name_index', 'isVal']].drop_duplicates()
name_df = name_df.sample(frac=1, random_state=SEED)

train_df = name_df[name_df['isVal']==False]
val_df = name_df[name_df['isVal']==True]

for slice_df in [train_df, val_df]:
    for idx, df in enumerate(np.array_split(slice_df, NUM_FOLDS)):
        df['cv_fold'] = idx
        tmp_df = tmp_df.append(df)

Y_df = meta_tmp[meta_tmp['name_index']=="Y"][['name_index']].drop_duplicates()        
Y_df['isVal'] = True
Y_df['cv_fold'] = 999

tmp_df = tmp_df.append(Y_df)

meta_new = meta_tmp.merge(tmp_df[['name_index','cv_fold']], on='name_index', how='left')
meta_new['split'] = meta_new['isVal'].apply(lambda x: "2nd_level" if x else "1st_level")
meta_new['version'] = datetime.date.today()
meta_new = meta_new.drop_duplicates()
## Visualize the counts
tmp = meta_new.groupby(['split','cv_fold']).size().to_frame('counts')
tmp
## Check that we didn't violate any of the rules 
## A) all videos with same original must belong to exactly one cv_fold
assert meta_new.groupby(['name_index']).nunique()['cv_fold'].max()==1

## B) all videos with same cluster_1 must belong to exactly one cv_fold
assert meta_new[meta_new.cv_fold!=999].groupby(['cluster_1']).nunique()['cv_fold'].max()==1

## C) length of the new metadata must match the length of the original data
assert len(meta_new)==orig_len
for col in [None, 'original', 'cluster_1','cluster_2']:
    df = meta_new.copy()
    values = df[col] if col!=None else None
    aggfunc = 'count' if col!=None else None
    
    print("\ncounts wrt "+col if col!=None else "")
    
    display(pd.crosstab([df['split'], df['cv_fold']],
                        [df['split'], df['cv_fold']],
                        values, aggfunc=aggfunc
                       ).fillna(0), 
           )
    print("- "*40)
# update person count and merge classes
meta_new = meta_new.merge(fake_types_df[['index',
                                         'first_person_frame_labels',
                                         'second_person_frame_labels']], on='index')

# update person count
meta_new['person_count'] = np.where(fake_types_df['first_person_label'].isna(),
                                    0, # no persons
                                    np.where(fake_types_df['second_person_label'].isna(),
                                             1, # only first
                                             2)) # first and second
meta_new.head()
## Save the splits
meta_new.sort_index(inplace=True)
meta_new = meta_new[KEEP_COLS]
meta_new.to_csv('../data/metadata_tr_val_split_folds.csv', index=False)
meta_new.head()
sns.countplot(x='person_count',hue='isVal',data=meta_new)
plt.suptitle('Person counts in train and val')
plt.show()

sns.countplot(x='person_count',hue='cv_fold',data=meta_new[meta_new['isVal']==False])
plt.suptitle('Person counts in training folds')
plt.show()

sns.countplot(x='person_count',hue='cv_fold',data=meta_new[meta_new['isVal']==True])
plt.suptitle('Person counts in validation folds')
plt.show()
sns.countplot(x='first_person_label',hue='isVal',data=meta_new)
plt.suptitle('First person fake-label in train and val')
plt.show()

sns.countplot(x='first_person_label',hue='cv_fold',data=meta_new[meta_new['isVal']==False])
plt.suptitle('First person fake-label in training folds')
plt.show()

sns.countplot(x='first_person_label',hue='cv_fold',data=meta_new[meta_new['isVal']==True])
plt.suptitle('First person fake-label in validation folds')
plt.show()<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup