<jupyter_start><jupyter_text>We will be performing all the below steps in Feature Engineering    1. Missing Values    2. Temporal Variable    3. Categorical Variable: Remove Rare Labels    4. Standardise the value of the variables to the same range<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
pd.pandas.set_option('display.max_columns',None)
dataset = pd.read_csv('data/train.csv')
dataset.head()
dataset.shape
# Always remember that there may be change of data leakage so, we need to split the data first and then perform feature
# Engineering
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(dataset, dataset['SalePrice'], test_size = 0.1,random_state = 0)
X_train.shape, X_test.shape<jupyter_output><empty_output><jupyter_text>Missing Values<jupyter_code>feature_nan = [feature for feature in dataset.columns if dataset[feature].isnull().sum() > 1 and dataset[feature].dtypes == 'O']
for feature in feature_nan:
    print(feature+ " : ",np.round(dataset[feature].isnull().mean(),4))
def replace_cat_feature(dataset,feature_nan):
    data = dataset.copy()
    data[feature_nan] = data[feature_nan].fillna('Missing')
    return data
dataset = replace_cat_feature(dataset,feature_nan)
dataset[feature_nan].isnull().sum()
dataset.head()
# Checking Numerical variables containing missing values
numerical_with_nan = [feature for feature in dataset.columns if dataset[feature].isnull().sum()> 1 
                      and dataset[feature].dtypes !='O' ]
for feature in numerical_with_nan:
    print(feature,np.round(dataset[feature].isnull().mean(),4))
for feature in numerical_with_nan:
    median_value = dataset[feature].median()
    dataset[feature+'nan'] = np.where(dataset[feature].isnull,1,0)
    dataset[feature].fillna(median_value,inplace = True)
dataset[numerical_with_nan].isnull().sum()
dataset.head()
# Date-Time Variable
for feature in dataset[['YearBuilt', 'YearRemodAdd', 'GarageYrBlt']]:
    dataset[feature] = dataset['YrSold'] - dataset[feature]
dataset.head()
dataset[['YearBuilt', 'YearRemodAdd', 'GarageYrBlt']].head()<jupyter_output><empty_output><jupyter_text>Numerical Variable's<jupyter_code># Found it using Exploratory Data Analysis 
num_features = [['LotFrontage','LotArea','1stFlrSF','GrLivArea','SalePrice']]
for feature in num_features:
    dataset[feature] = np.log(dataset[feature])
    
dataset.head()<jupyter_output><empty_output><jupyter_text>Handling Rare Categorical Feature<jupyter_code># We will remove categorical variables that are present less than 1% of the observations
categorical_feature = [feature for feature in dataset.columns if dataset[feature].dtypes == 'O']
categorical_feature
for feature in categorical_feature:
    temp = dataset.groupby(feature)['SalePrice'].count()/len(dataset) # getting percentage of that feature
    temp_df = temp[temp > 0.01].index
    dataset[feature] = np.where(dataset[feature].isin(temp_df),dataset[feature],'Rare_var')
dataset.head(100)<jupyter_output><empty_output><jupyter_text>Feature Scaling<jupyter_code>for feature in categorical_feature:
    labels_ordered=dataset.groupby([feature])['SalePrice'].mean().sort_values().index
    labels_ordered={k:i for i,k in enumerate(labels_ordered,0)}
    dataset[feature]=dataset[feature].map(labels_ordered)

scaling_feature=[feature for feature in dataset.columns if feature not in ['Id','SalePrice'] ]
len(scaling_feature)
scaling_feature
feature_scale=[feature for feature in dataset.columns if feature not in ['Id','SalePrice']]

from sklearn.preprocessing import MinMaxScaler
scaler=MinMaxScaler()
scaler.fit(dataset[feature_scale])
scaler.transform(dataset[feature_scale])
# Transform the train and test set, and add on the Id and SalePrice variables.
data = pd.concat([dataset[['Id','SalePrice']].reset_index(drop = True),
          pd.DataFrame(scaler.transform(dataset[feature_scale]), columns = feature_scale)],
              axis = 1)
data.head()
data.to_csv('X_train.csv', index = False)
X_test<jupyter_output><empty_output>