# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: pytorch38
#     language: python
#     name: pytorch38
# ---

# +
import pandas as pd
import numpy as np
import copy

import seaborn as sns
import matplotlib as mpl
import matplotlib.patches as patches
import matplotlib.pyplot as plt

from unidip import UniDip
import unidip.dip as dip
# -

pd_train = pd.read_csv('../data/train.csv')
pd_train.shape


# ## DIP test hypothesis H0:uni-modal vs H1:not uni-modal
# ## UniDip returns list of tuples: each tuple containing the start and end indecies of the distribution

def test_modality(values):
    data = np.msort(values)
    res1 = dip.diptst(data)
    res2 = UniDip(data).run()
    return (res1, res2)


# +
# Create segments between 5 and 15

def create_segments(total_bins, bin_len):
    bins = [(0, bin_len)]
    for seg in range(total_bins-1):
        if seg == total_bins-2:
            bins.append((bins[-1][1], np.nan))
        else:
            bins.append((bins[-1][1], bins[-1][1]+bin_len))
    return bins

def create_bins(df, column, num_of_bins=5):
    bin_len = int(len(df)/num_of_bins)
    bins = create_segments(num_of_bins, bin_len)
    df = df.sort_values(column, ascending=True).reset_index(drop=True)
    column_name = column+"_segment"
    df[column_name] = 0
    for index, seg in enumerate(bins):
        start = seg[0]
        end = seg[1]
        if end is not np.nan:
            df.loc[start:end, column_name] = str(index)
        else:
            df.loc[start:, column_name] = str(index)
    return df


# +
# number of bins are selected by dip analysis
# 4 bins gives the lowest neutral samples a.k.a error
# error: the samples cannot be placed in either group [B1 or B2] they are in the middle between two modals (end index of modal1 - start index of modal2)

bins = 4
pd_train = create_bins(copy.deepcopy(pd_train), "standard_error", bins)
# -

for i in range(bins):
    print(test_modality(pd_train.loc[pd_train['standard_error_segment'] == str(i), 'target'].values))

plt.figure(figsize = (25, 11))
sns.kdeplot(pd_train["target"], hue=pd_train["standard_error_segment"])
plt.title("Target Distribution (segmented)", size=25)
plt.xlabel("Value", size=20)
plt.ylabel("Frequency", size=20)

# ## Let us look at the neutral samples

bi_modal_1 = pd_train.loc[pd_train['standard_error_segment'] == str(3)].sort_values(by='target').reset_index(drop=True)

# +
# Everything upto 327 is in B1 and everythong from 443 is B2
# Let us see how we will assign B1 or B2 for lables between 327 and 443

test_modality(bi_modal_1.iloc[327:443]['target'].values)
# -

# ## The Dip test is not significant, the data is unimodal. Divide them at the center and assign the first half to B1 and second half to B2

# + active=""
#
# -

# ## generating class_labels map based on the bins. We will use target as key because they are unique for every row.
# ### If not for target we would have had to introduced a id column in clenaing_and_adding_new_samples_using_translate.py and use that id column here.

G_keys = pd_train.loc[(pd_train['standard_error_segment'] == '0') | (pd_train['standard_error_segment'] == '1') | (pd_train['standard_error_segment'] == '2'), 'target'].values
B0_keys = bi_modal_1[:355]['target'].values
B1_keys = bi_modal_1[355:]['target'].values

len(G_keys)+len(B0_keys)+len(B1_keys) == pd_train.shape[0]


# +
def assign_dev_1_labels(row):
    if row['target'] in G_keys:
        return 'G'
    else: return 'B'

pd_train['dev_1'] = pd_train.apply(lambda row: assign_dev_1_labels(row), axis=1)

pd_train.value_counts(['dev_1'], normalize=True)


# +
def assign_dev_2_labels(row):
    if row['target'] in G_keys:
        return 'G'
    elif row['target'] in B0_keys:
        return 'B0'
    else: return 'B1'

pd_train['dev_2'] = pd_train.apply(lambda row: assign_dev_2_labels(row), axis=1)

pd_train.value_counts(['dev_2'], normalize=True)
# -

train = pd.read_csv('../data/new_preprocessed_train_data.csv', low_memory=False)


# +
def assign_dev_1_labels_for_dup(row):
    if row['labels'] in G_keys:
        return 'G'
    else: return 'B'

train['dev_1'] = train.apply(lambda row: assign_dev_1_labels_for_dup(row), axis=1)

train.value_counts(['dev_1'], normalize=True)


# +
def assign_dev_2_labels_for_dup(row):
    if row['labels'] in G_keys:
        return 'G'
    elif row['labels'] in B0_keys:
        return 'B0'
    else: return 'B1'

train['dev_2'] = train.apply(lambda row: assign_dev_2_labels_for_dup(row), axis=1)

train.value_counts(['dev_2'], normalize=True)
# -

train.loc[train['dev_2'] == 'G'].describe().T

train.loc[train['dev_2'] == 'B0'].describe().T

train.loc[train['dev_2'] == 'B1'].describe().T

train.describe().T

train.to_csv('../data/classifier_training_data/raw_data.csv', index=False)



StarcoderdataJupyterScriptsDedupFiltered