<jupyter_start><jupyter_text><jupyter_code>#### 1)Mean/Median/Mode Imputation
#It is easy to implement and fastest way to obtain a complete dataset
#But, it imapcts the correlation and also distortion is in the original variance
import pandas as pd
from google.colab import files
uploaded = files.upload()
df=pd.read_csv("titanic.csv", usecols=['Age', 'Fare', 'Survived'])
df.head()
df.isnull().mean()
def impute_nan(df,variable,median):
    df[variable+"_median"]=df[variable].fillna(median)
median = df.Age.median()
median
impute_nan(df, 'Age', median)
df.head()
print(df['Age'].std())
print(df['Age_median'].std())
import matplotlib.pyplot as plt
%matplotlib inline
fig = plt.figure()
ax = fig.add_subplot(111)
df['Age'].plot(kind='kde', ax=ax)
df.Age_median.plot(kind='kde', ax=ax, color='red')
lines, labels = ax.get_legend_handles_labels()
ax.legend(lines, labels, loc='best');
#### 2)Random Sample Imputation
#It consist of random observation from dataset and use the random observation to replace the nan value.
#There is less distortion in variance in Random Sample imputation but it cannot be used everywork.
#It assumes that data is missing completely at random.
import pandas as pd
df=pd.read_csv('titanic.csv', usecols=['Age','Fare','Survived'])
df.head()
df.isnull().mean()
df.isnull().sum()
df['Age'].dropna().sample(df['Age'].isnull().sum(), random_state=0)
def impute_nan(df,variable,median):
    df[variable+"_median"]=df[variable].fillna(median)
    df[variable+"random"]=df[variable]
    random_sample = df[variable].dropna().sample(df[variable].isnull().sum(), random_state=0)
    random_sample.index=df[df[variable].isnull()].index
    df.loc[df[variable].isnull(),variable+'_random']=random_sample
median = df.Age.median()
median
impute_nan(df,"Age",median)
df.head()
import matplotlib.pyplot as plt
%matplotlib inline
fig = plt.figure()
ax = fig.add_subplot(111)
df['Age'].plot(kind='kde', ax=ax)
df.Age_median.plot(kind='kde', ax=ax, color='red')
df.Age_random.plot(kind='kde', ax=ax, color='green')
lines, labels = ax.get_legend_handles_labels()
ax.legend(lines, labels, loc='best')
#### 3)Capturing NAN values with a new feature
#Works when the data is not missing completely at random.
#creates addition features which can leads to curse of dimentionality
df=pd.read_csv('titanic.csv', usecols=['Age','Fare','Survived'])
df.head()
import numpy as np
df['Age_NAN']=np.where(df['Age'].isnull(),1,0)
df.head()
df.Age.median()
df['Age'].fillna(df.Age.median(),inplace=True)
#### 4)End of Distribution Imputation
#MCAR
df=pd.read_csv('titanic.csv', usecols=['Age','Fare','Survived'])
df.head()
df.Age.hist(bins=50);
extreme=df.Age.mean()+3*df.Age.std()
import seaborn as sns
sns.boxplot('Age',data=df);
def impute_nan(df,variable,median,extreme):
    df[variable+"_end_distribution"]=df[variable].fillna(extreme)
    df[variable].fillna(median,inplace=True)
impute_nan(df,'Age',df.Age.median(),extreme)
df.head()
df['Age'].hist(bins=50);
df['Age_end_distribution'].hist(bins=50);
sns.boxplot('Age_end_distribution',data=df);
#### 5)Arbitrary Value Imputation 
 #It is use to find out the important of missing value.Rarely use.
import pandas as pd
df=pd.read_csv("titanic.csv", usecols=["Age","Fare","Survived"])
df.head()
def impute_nan(df,variable):
    df[variable+'_zero']=df[variable].fillna(0)
    df[variable+'_hundred']=df[variable].fillna(100)
df['Age'].hist(bins=50);<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup