<jupyter_start><jupyter_text>1. Load data<jupyter_code># Load csv file
df_train = pd.read_csv('../data_csv/aug_train.csv')
df_test = pd.read_csv('../data_csv/aug_test.csv')
test_target = np.load('../data_csv/jobchange_test_target_values.npy')
test_target = pd.DataFrame(test_target,columns=['target'])
test_target['enrollee_id'] = df_test['enrollee_id']
test_target = test_target[['enrollee_id','target']]
test_target.to_csv('../data_csv/test_target.csv')
# test_target = pd.read_csv('../data_csv/test_target.csv')
test_target
# df
# Check each column
# terminal install: conda install -c conda-forge pandas-profiling
from pandas_profiling import ProfileReport as pr
profile = pr(df_train, minimal=True).to_notebook_iframe()<jupyter_output><empty_output><jupyter_text>2. Examine and impute missing values<jupyter_code>df_train.info()
# Pairplot
sns.pairplot(df_train, corner=True, height=1.5, plot_kws={'size': 3}, hue='target');
# Examine data
df_train['company_type'].value_counts()

df_train['enrolled_university'].value_counts()

df_train['education_level'].value_counts()

df_train['experience'].value_counts()

df_train['company_size'].value_counts()

df_train['company_type'].value_counts()

df_train['last_new_job'].value_counts()
# Replace string with float/int
df_train['experience'] = df_train['experience'].replace('>20','25')
df_train['experience'] = df_train['experience'].replace('<1','0.5')
df_train['experience'] = df_train['experience'].astype('float')
df_train['last_new_job'] = df_train['last_new_job'].replace('>4','5')
df_train['last_new_job'] = df_train['last_new_job'].replace('never','0')
# Impute/fill NaN
df_train['gender'] = df_train['gender'].replace(np.nan, 'unknown')
df_train['enrolled_university'] = df_train['enrolled_university'].replace(np.nan, 'unknown')
df_train['education_level'] = df_train['education_level'].replace(np.nan, 'unknown')
df_train['major_discipline'] = df_train['major_discipline'].replace(np.nan, 'unknown')
df_train['education_level'] = df_train['education_level'].replace(np.nan, 'unknown')
df_train['experience'] = df_train['experience'].fillna(value = df_train['experience'].median())
df_train['company_size'] = df_train['company_size'].fillna(value = df_train['company_size'].value_counts().index[0])
df_train['company_type'] = df_train['company_type'].replace(np.nan, 'unknown')
df_train['last_new_job'] = df_train['last_new_job'].fillna(value = df_train['last_new_job'].median()).astype('int')
df_train['target'] = df_train['target'].astype('int')
df_train.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 19158 entries, 0 to 19157
Data columns (total 14 columns):
 #   Column                  Non-Null Count  Dtype  
---  ------                  --------------  -----  
 0   enrollee_id             19158 non-null  int64  
 1   city                    19158 non-null  object 
 2   city_development_index  19158 non-null  float64
 3   gender                  19158 non-null  object 
 4   relevent_experience     19158 non-null  object 
 5   enrolled_university     19158 non-null  object 
 6   education_level         19158 non-null  object 
 7   major_discipline        19158 non-null  object 
 8   experience              19158 non-null  float64
 9   company_size            19158 non-null  object 
 10  company_type            19158 non-null  object 
 11  last_new_job            19158 non-null  int64  
 12  training_hours          19158 non-null  int64  
 13  target                  19158 non-null  int64  
dtypes: float64(2), int64(4), object(8)
mem[...]<jupyter_text>3. Pickle<jupyter_code>df_train.to_pickle('../dump/df_train.csv')<jupyter_output><empty_output><jupyter_text>4. Repeat for test set  Examine and impute missing values<jupyter_code>df_test['target'] = test_target
df_test.info()
# Pairplot
sns.pairplot(df_test, corner=True, height=1.5, plot_kws={'size': 3}, hue='target');
# Examine data
df_train['company_type'].value_counts()

df_train['enrolled_university'].value_counts()

df_train['education_level'].value_counts()

df_train['experience'].value_counts()

df_train['company_size'].value_counts()

df_train['company_type'].value_counts()

df_train['last_new_job'].value_counts()
# Replace string with float/int
df_test['experience'] = df_test['experience'].replace('>20','25')
df_test['experience'] = df_test['experience'].replace('<1','0.5')
df_test['experience'] = df_test['experience'].astype('float')
df_test['last_new_job'] = df_test['last_new_job'].replace('>4','5')
df_test['last_new_job'] = df_test['last_new_job'].replace('never','0')
# Impute/fill NaN
df_test['gender'] = df_test['gender'].replace(np.nan, 'unknown')
df_test['enrolled_university'] = df_test['enrolled_university'].replace(np.nan, 'unknown')
df_test['education_level'] = df_test['education_level'].replace(np.nan, 'unknown')
df_test['major_discipline'] = df_test['major_discipline'].replace(np.nan, 'unknown')
df_test['education_level'] = df_test['education_level'].replace(np.nan, 'unknown')
df_test['experience'] = df_test['experience'].fillna(value = df_test['experience'].median())
df_test['company_size'] = df_test['company_size'].fillna(value = df_test['company_size'].value_counts().index[0])
df_test['company_type'] = df_test['company_type'].replace(np.nan, 'unknown')
df_test['last_new_job'] = df_test['last_new_job'].fillna(value = df_test['last_new_job'].median()).astype('int')
df_test['target'] = df_test['target'].astype('int')
df_test.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 2129 entries, 0 to 2128
Data columns (total 14 columns):
 #   Column                  Non-Null Count  Dtype  
---  ------                  --------------  -----  
 0   enrollee_id             2129 non-null   int64  
 1   city                    2129 non-null   object 
 2   city_development_index  2129 non-null   float64
 3   gender                  2129 non-null   object 
 4   relevent_experience     2129 non-null   object 
 5   enrolled_university     2129 non-null   object 
 6   education_level         2129 non-null   object 
 7   major_discipline        2129 non-null   object 
 8   experience              2129 non-null   float64
 9   company_size            2129 non-null   object 
 10  company_type            2129 non-null   object 
 11  last_new_job            2129 non-null   int64  
 12  training_hours          2129 non-null   int64  
 13  target                  2129 non-null   int64  
dtypes: float64(2), int64(4), object(8)
memor[...]<jupyter_text>3. Pickle<jupyter_code>df_test.to_pickle('../dump/df_test.csv')<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup