# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

# +
# DATA MUNGING

# Missing Value Treatment
# Outliers

# ISSUES
# Inaccurate analysis
# Modeling won't work in many cases

# SOLUTIONS
# Deletion
# Imputation

# +
import numpy as np
import pandas as pd
import os

raw_data_path=os.path.join(os.path.pardir,'data','raw')
train_file_path=os.path.join(raw_data_path,'train.csv')
test_file_path=os.path.join(raw_data_path,'test.csv')

train_df=pd.read_csv(train_file_path,index_col="PassengerId")
test_df=pd.read_csv(test_file_path,index_col="PassengerId")

test_df["Survived"]=-888

df=pd.concat((train_df,test_df),axis=0)

df.info()
# -

df[df.Embarked.isnull()]

df.Embarked.value_counts()

pd.crosstab(df[df.Survived!=-888].Survived,df[df.Survived!=-888].Embarked)

# df.Embarked.fillna('S',inplace=True)


df.groupby(['Pclass','Embarked']).Fare.median()

df.Embarked.fillna('C',inplace=True)

df[df.Embarked.isnull()]

df.info()

df[df.Fare.isnull()]

df[df.Age.isnull()]

df.Age.plot(kind="hist",bins=20,color='c');

df.Age.median()

df.Age.mean()

pd.options.display.max_rows=15

df[df.Age.isnull()]

df.groupby('Sex').median()

df[df.Age.notnull()].boxplot('Age','Sex')

df.groupby("Sex").Age.transform('median')

df[df.Age.notnull()].boxplot('Age','Pclass')

df.Name


def GetTitle(name):
    first_name_with_title=name.split(',')[1]
    title=first_name_with_title.split('.')[0]
    title=title.strip().lower()
    return title



df['Title']=df.Name.map(lambda x: GetTitle(x))

df[df.Age.notnull()].boxplot('Age','Title');

title_age_median=df.groupby('Title').Age.transform('median')

df.Age.fillna(title_age_median,inplace=True)

df.info()


df[df.Fare.isna()]

df[df.Fare.notnull()].groupby('Pclass').Fare.median()

fare_median=df.groupby('Pclass').Fare.transform('median');

df.Fare.fillna(fare_median)

df.info()

# %%HTML
<h1> WORKING WITH OUTLIERS </h1>

df.Age.plot(kind='hist', bins=20)

df[df.Age>70]

df.Fare.plot(kind='hist',bins=20)

df.Fare.plot(kind='box')

df.loc[df.Fare==df.Fare.max()]

LogFare=np.log(df.Fare+1.0)

LogFare.plot(kind='hist',bins=20);

df.Fare.plot(kind='hist',bins=20);

# %%HTML
<h1>Binning</h1>

pd.qcut(df.Fare,4,labels=['very_low','low','high','very_high'])

pd.qcut(df.Fare,4,labels=['very_low','low','high','very_high']).value_counts().plot(kind='bar');

df['Fare_Bin']=pd.qcut(df.Fare,4,labels=['very_low','low','high','very_high'])

df


# %%HTML
<h1> FEATURE ENGINEERING</h1>

# +
# TRANSFORMATION
# CREATION
# SELECTION
# Domain_Knowledge + Technical_Expertise
# -

df['AgeState']=np.where(df.Age>=18,'Adult','Child')

df.AgeState.value_counts()


pd.crosstab(df.Survived[df.Survived!=-888],df.AgeState[df.Survived!=-888])

df['Family_Size']=df.SibSp+df.Parch+1

df

df.Family_Size.plot(kind='hist')

df[df.Family_Size>8]

pd.crosstab(df[df.Survived!=-888].Family_Size,df[df.Survived!=-888].Survived)

df['isMother']=np.where(((df.Sex=='female') & (df.Parch>0) & (df.Title!='miss')),1,0)

df[df.isMother==1][['Age']].plot(kind='hist')

pd.crosstab(df[df.Survived!=-888].Survived,df[df.Survived!=-888].isMother)

df.Cabin

df.Cabin.unique()

df.loc[df.Cabin=='T','Cabin']=np.NAN


def get_deck(cabin):
    return np.where(pd.notnull(cabin),str(cabin)[0].upper(),'Z')


df["Deck"]=df.Cabin.map(lambda x:get_deck(x))

df['Deck'].unique()

pd.crosstab(df[df.Survived!=-888].Deck,df[df.Survived!=-888].Survived)

df.info()

# %%HTML
<h1>CATEGORICAL FEATURE ENCODING</h1>
<h2>BINARY ENCODING</h2>
<h2>LABEL ENCODING</h2>
<h2>ONE HOT ENCODING</h2>

df.Embarked

# %%HTML
<h2>CATEGORICAL FEATURE ENCODING</h2>

df['isMale']=np.where(df.Sex=='male',1,0)

df=pd.get_dummies(df,columns=['Deck','Pclass','Title','Fare_Bin','Embarked','AgeState']);

df.info()

df.Ticket

df.drop(['Cabin','Name','Ticket','Parch','SibSp','Sex'],axis=1,inplace=True)

columns=[column for column in df.columns if column!='Survived']
columns=['Survived']+columns
df=df[columns]

df.info()


# %%HTML
<h1>SAVE PROCESSED DATA</h1>

processed_data_path=os.path.join(os.path.pardir,'data','processed')
write_train_path=os.path.join(processed_data_path,'train.csv')
write_test_path=os.path.join(processed_data_path,'test.csv')

df.loc[df.Survived!=-888].to_csv(write_train_path)
df.loc[df.Survived==-888].to_csv(write_test_path)

# %%HTML
<h2> Building the data processing script</h2>

#to be done later


# + language="html"
# <h1> ADVANCED VISUALIZATIONS USING MAT</h1>
# -

import matplotlib.pyplot as plt
# %matplotlib inline

plt.hist(df.Age,bins=20,color='c');
plt.title('Histogram : Age')
plt.xlabel('Bins')
plt.ylabel('Counts')

f,ax=plt.subplots()
ax.hist(df.Age,bins=20,color='c')
ax.set_xlabel('Bins')
ax.set_ylabel('Counts')
plt.show()

# +
f,(ax1,ax2)=plt.subplots(1,2,figsize=(14,3))
ax1.hist(df.Age,bins=20,color='c')
ax1.set_xlabel('Bins')
ax1.set_ylabel('Counts')
ax1.set_title('Histogram : Age')

ax2.hist(df.Family_Size,bins=20,color='tomato')
ax2.set_xlabel('Bins')
ax2.set_ylabel('Counts')
ax2.set_title('Histogram : Fare')

plt.show()
# -

df.Fare

# +
#Multiple plots can be made using array indexing, to avoid overlapping use plt.tight_layout()

# +
# remove a plot -> ax[2,2].axis('off')
# -



StarcoderdataJupyterScriptsDedupFiltered