<jupyter_start><jupyter_text>Generating new variables<jupyter_code># compares variables and generates a new one
def compare_human(x):
    if x["DISAPPROVALS_BL"] == 0 and x["DISAPPROVALS_EN"] == 0:
        return(0)
    else:
        return(1)
df["DISAPPROVALS_HUMAN"] = df.apply(compare_human, axis= 1)
def compare_exacts(x):
    if x["DISAPPROVALS_FM"] == 0 and x["DISAPPROVALS_OM"] == 0:
        return(0)
    else:
        return(1)
df["DISAPPROVALS_EXACTS"] = df.apply(compare_exacts, axis= 1)
# Viewing new data frame
df
# Converting categorical variable into numeric variable
var = ['NOTE_BL_CAT', 'NOTE_EN_CAT', 'NOTE_FM_CAT', 'NOTE_OM_CAT']
count = 0
for i in var:
    df[count]= df[i].map({"(-1, 2]": 2,
                           "(2, 4]": 4, 
                           "(4, 6]": 6,
                           "(6, 8]": 8,
                           "(8, 10]": 10})
    count += 1
df.rename(columns={0:'NOTE_BL_CAT_N', 1:'NOTE_EN_CAT_N', 2:'NOTE_FM_CAT_N', 3:'NOTE_OM_CAT_N'}, inplace=True) 
# checking data type 
df.dtypes
# Arranging dataframe columns
a = ['DISAPPROVALS_BL', 'DISAPPROVALS_EN', 'DISAPPROVALS_HUMAN', 'DISAPPROVALS_FM','DISAPPROVALS_OM', 
     'DISAPPROVALS_EXACTS', 'NOTE_BL', 'NOTE_EN', 'NOTE_FM', 'NOTE_OM', 'NOTE_BL_CAT', 'NOTE_EN_CAT',
     'NOTE_FM_CAT', 'NOTE_OM_CAT', 'NOTE_BL_CAT_N', 'NOTE_EN_CAT_N', 'NOTE_FM_CAT_N', 'NOTE_OM_CAT_N', 
     'ENGLISH', 'H_CLASS_PRES', 'ONLINE_TASKS', 'ABSENCES', 'PROFILE_N']
df = df.reindex(columns= a)
df.shape
df.isna().sum()
# Saving data
df.to_csv("Data/data3.csv", index= False)
# creating new dataframe and removing from categorical variables 
df1 = df.drop(['NOTE_BL_CAT', 'NOTE_EN_CAT', 'NOTE_FM_CAT', 'NOTE_OM_CAT'], axis= 1)
df1
df1.shape
# Checking data type
df1.dtypes
df1.isna().sum()<jupyter_output><empty_output><jupyter_text>Applying feature selection<jupyter_code>x = df1.iloc[:, 0:18]
y = df1.PROFILE_N
x.shape
y.shape
feature_selection = pd.DataFrame(SelectKBest(chi2, k= 10).fit_transform(x, y))
feature_selection["TARGET"]= y
feature_selection.head(15)
# saving feature selection in csv format
feature_selection.to_csv("Data/selection_variables.csv", index= False)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup