# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Problem Statement

#     A US-based housing company named Surprise Housing has decided to enter the Australian market. The company uses data analytics to purchase houses at a price below their actual values and flip them on at a higher price. For the same purpose, the company has collected a data set from the sale of houses in Australia. The data is provided in the CSV file below.
#
#
#
#     The company is looking at prospective properties to buy to enter the market. You are required to build a regression model using regularisation in order to predict the actual value of the prospective properties and decide whether to invest in them or not.
#
#
#
#     The company wants to know:
#
#     Which variables are significant in predicting the price of a house, and
#
#     How well those variables describe the price of a house.

# ## Loading important libraries

# +
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn import linear_model, metrics
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Ridge
from sklearn.linear_model import Lasso
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import mean_squared_error, r2_score

import os

# hide warnings
import warnings
warnings.filterwarnings('ignore')

# -

house = pd.read_csv('train.csv')

house.head()

house.info()

# listing all missing values percentage wise
(house.isnull().sum() * 100 / len(house)).sort_values(ascending=False).head()

# +
# Counting unique value in each categorical column
cat_col =house.select_dtypes(include=['object'])

for i in list(cat_col.columns):
    print("We have {} unique value in {} column: {}.".format(len(cat_col[i].unique()), 
                                                     i, cat_col[i].unique()))
    print('*'*140)
# -

# ## analysing the target feature , i.e SalePrice

house['SalePrice'].describe()

#histogram
sns.distplot(house['SalePrice']);


#skewness and kurtosis
print("Skewness: %f" % house['SalePrice'].skew())
print("Kurtosis: %f" % house['SalePrice'].kurt())

# ## Relationship with numerical variables

# #### some important numerical feature which can affect the house price significantly based on business requirements
#     GrLivArea,PoolArea,GarageArea,OpenPorchSF,LotArea,TotalBsmtSF,MasVnrArea

#scatter plot grlivarea/saleprice
var = 'GrLivArea'
data = pd.concat([house['SalePrice'], house[var]], axis=1)
data.plot.scatter(x=var, y='SalePrice', ylim=(0,800000));


# Scatter plot between target feature(SalePrice) and the important features
for col in ('GrLivArea','PoolArea','GarageArea','OpenPorchSF','LotArea','TotalBsmtSF','MasVnrArea'):
    data = pd.concat([house['SalePrice'], house[col]], axis=1)
    data.plot.scatter(x=col, y='SalePrice', ylim=(0,800000),title = "SalePrice vs {}".format(col));


# ## Data Cleaning and feature engineering

#check missing ratios
total_data_na = ((house.isnull().sum() / len(house)) * 100).sort_values(ascending=False)

#check missing ratios top 30
total_data_na = total_data_na.drop(total_data_na[total_data_na == 0].index).sort_values(ascending=False)[:30]
total_data_na


# +
# plotting missing values
plt.subplots(figsize=(15,12))
plt.xticks(rotation='90')
sns.barplot(x=total_data_na.index, y=total_data_na)

plt.xlabel('Features', fontsize=15)
plt.ylabel('Percent of missing value', fontsize=15)
plt.title('Percent missing data by feature', fontsize=25)
# -

# dropping columns having Nan values more than 75%
house = house.loc[:, house.isnull().mean() < .75]


# FireplaceQu - NA means no fireplace, so replacing with None
house["FireplaceQu"] = house["FireplaceQu"].fillna("None")

# ## Imputation according to the required DataType and Data_description text file

# +
# for LotFrontage, replacing missing value with median
house["LotFrontage"]=house.groupby("Neighborhood")["LotFrontage"].transform(
    lambda x: x.fillna(x.median()))

# replacing Na values to None because Na means there is no facility for Garage
for col in ('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
    house[col] =house[col].fillna('None')
    
# Replacing missing with 0 (No garage = no cars in such garage)
for col in ('GarageYrBlt', 'GarageArea', 'GarageCars'):
    house[col] = house[col].fillna(0)

# Missing value in these feature means no basement (replace with 0)
for col in ('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
    house[col] = house[col].fillna(0)
    
# NaN means in these column is no basement
for col in ('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
    house[col] = house[col].fillna('None')
    
# NA means no masonry veneer for houses (Fill zero for area and None for type)
house["MasVnrType"] = house["MasVnrType"].fillna("None")
house["MasVnrArea"] = house["MasVnrArea"].fillna(0)


# Filling MSZoning with most common value 'RL'
house['MSZoning'] = house['MSZoning'].fillna(house['MSZoning'].mode()[0])


# For Utilitites, most of the records are 'AllPub'. It won't help in prediction
house = house.drop(['Utilities'], axis=1)


# data description says- Assume typical(Typ) unless deductions are warranted
house["Functional"] = house["Functional"].fillna("Typ")


# replacing with most repeated "SBrkr" value
house['Electrical'] = house['Electrical'].fillna(house['Electrical'].mode()[0])

# set "TA" most frequent in place of missing value
house['KitchenQual'] = house['KitchenQual'].fillna(house['KitchenQual'].mode()[0])


# replacing with most common value 
house['Exterior1st'] = house['Exterior1st'].fillna(house['Exterior1st'].mode()[0])
house['Exterior2nd'] = house['Exterior2nd'].fillna(house['Exterior2nd'].mode()[0])


# replacing with most common value again
house['SaleType'] = house['SaleType'].fillna(house['SaleType'].mode()[0])

# MSSubClass-type of dwelling, NA means no building class. (fill with None)
house['MSSubClass'] = house['MSSubClass'].fillna("None")

# -

# ## Now checking the missing values
#

((house.isnull().sum()/len(house))*100).sort_values(ascending=False).head()

# ### converting data types of some columns whcih are int but it should be str

# +
#MSSubClass=The building class
house['MSSubClass'] = house['MSSubClass'].apply(str)


#Changing OverallCond into a categorical variable
house['OverallCond'] = house['OverallCond'].astype(str)

# -

# ### Visualizing the cleaned dataset

df_cat = house.select_dtypes(include = 'object')
#analysis of categorical variables 
plt.figure(figsize=(30,100))
for i in enumerate(df_cat.columns.values):
    sns.set_style("whitegrid")
    plt.subplot(12,4,i[0]+1)
    sns.countplot(i[1],data = df_cat)
    plt.xticks(rotation = 45,fontsize=13)
    plt.xlabel(i[1],fontsize = 15,fontweight = 'bold')
    plt.title('Sale_Price'+' vs '+i[1],fontweight = 'bold',fontsize=15)



house.head()

num_vars = house.select_dtypes(exclude=['object']).columns.tolist()

cat_cols = house.select_dtypes(include=['object']).columns.tolist()

# ## using dummies to convert categorical columns to numerical columns

# +
# get dummy variables for season, weekday, mnth and weathersit
dummy_vars = pd.get_dummies(house[cat_cols],drop_first=True)

# concat the dummy df with original df
house = pd.concat([house,dummy_vars], axis = 1)

# drop season column
house.drop(cat_cols, axis=1, inplace=True)
# -

house.shape

# ## Splitting data into train and test 

from sklearn.model_selection import train_test_split
# We specify this so that the train and test data set always have the same rows, respectively
# Split train test dataset
df_train, df_test = train_test_split(house, train_size = 0.7, random_state = 10 )
print(df_train.shape)
print(df_test.shape)


# ### Rescaling the Features
# We will use MinMax scaling.
#
# During EDA we could observe that there is different range of data in the data set. So it becomes important to scale the data. Here we will be using Min-Max scaling (normalisation) to scale both training and tesing dataset.

from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler()

# Scaling of train set
# instantiate an object
scaler = MinMaxScaler()
# fit and transform on training data
df_train[num_vars] = scaler.fit_transform(df_train[num_vars])
df_train.head()

# transform test dataset 
df_test[num_vars] = scaler.transform(df_test[num_vars])
df_test.head()

# distribution after scaling
plt.figure(figsize=(16,6))
plt.subplot(121)
sns.distplot(df_train.SalePrice)
plt.subplot(122)
sns.distplot(df_test.SalePrice)



# Creating X and y data dataframe for train set
y_train = df_train.pop('SalePrice')
X_train = df_train
X_train.head()

# +
# Creating X and y data dataframe for test set
y_test = df_test.pop('SalePrice')
X_test = df_test

X_test.head()

# -



# Define dictionary to store our rankings
ranks = {}
# Create our function which stores the feature rankings to the ranks dictionary
def ranking(ranks, names, order=1):
    minmax = MinMaxScaler()
    ranks = minmax.fit_transform(order*np.array([ranks]).T).T[0]
    ranks = map(lambda x: round(x,5), ranks)
    return dict(zip(names, ranks))


imp_col = X_train.columns.values

# ## Ridge and Lasso Regression
#
# Let's now try predicting house prices, a dataset used in simple linear regression, to perform ridge and lasso regression.

# # Ridge Regression

# +
# list of alphas to tune - if value too high it will lead to underfitting, if it is too low, 
# it will not handle the overfitting
params = {'alpha': [0.0001, 0.001, 0.01, 0.05, 0.1, 
 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 2.0, 3.0, 
 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 20, 50, 100, 500, 1000 ]}

ridge = Ridge()

# cross validation
folds = 5
model_cv = GridSearchCV(estimator = ridge, 
                        param_grid = params, 
                        scoring= 'neg_mean_absolute_error',  
                        cv = folds, 
                        return_train_score=True,
                        verbose = 1)            
model_cv.fit(X_train, y_train) 
# -

cv_result_r = pd.DataFrame(model_cv.cv_results_)
cv_result_r['param_alpha'] = cv_result_r['param_alpha'].astype('float32')
cv_result_r.head()


plt.figure(figsize=(16,8))
plt.plot(cv_result_r['param_alpha'],cv_result_r['mean_train_score'])
plt.plot(cv_result_r['param_alpha'],cv_result_r['mean_test_score'])
plt.xlabel('Alpha')
plt.xscale('log')
plt.ylabel('R2 Score')
plt.show()

# Printing the best hyperparameter alpha
print(model_cv.best_params_)

# +
#Fitting Ridge model for alpha = 10 and printing coefficients which have been penalised
alpha = 10
ridge = Ridge(alpha=alpha)

ridge.fit(X_train, y_train)
# print(ridge.coef_)

# +
# Lets calculate some metrics such as R2 score, RSS and RMSE
y_pred_train = ridge.predict(X_train)
y_pred_test = ridge.predict(X_test)

metric2 = []
r2_train_lr = r2_score(y_train, y_pred_train)
print(r2_train_lr)
metric2.append(r2_train_lr)

r2_test_lr = r2_score(y_test, y_pred_test)
print(r2_test_lr)
metric2.append(r2_test_lr)

rss1_lr = np.sum(np.square(y_train - y_pred_train))
print(rss1_lr)
metric2.append(rss1_lr)

rss2_lr = np.sum(np.square(y_test - y_pred_test))
print(rss2_lr)
metric2.append(rss2_lr)

mse_train_lr = mean_squared_error(y_train, y_pred_train)
print(mse_train_lr)
metric2.append(mse_train_lr**0.5)

mse_test_lr = mean_squared_error(y_test, y_pred_test)
print(mse_test_lr)
metric2.append(mse_test_lr**0.5)
# -

# ## Top 20 features from ridge 

rank_ridge = ranking(np.abs(ridge.coef_), imp_col)
sorted(rank_ridge.items(), key=lambda item: item[1] , reverse=True)[:20]

# ## According to Problem Statement - Part II What if we choose double value of alpha



# +
#Fitting Ridge model for alpha = 20 and printing coefficients which have been penalised
alpha2x = 20
ridge = Ridge(alpha=alpha2x)

ridge.fit(X_train, y_train)
# print(ridge.coef_)

# +
# Lets calculate some metrics such as R2 score, RSS and RMSE
y_pred_train = ridge.predict(X_train)
y_pred_test = ridge.predict(X_test)

metric2 = []
r2_train_lr = r2_score(y_train, y_pred_train)
print(r2_train_lr)
metric2.append(r2_train_lr)

r2_test_lr = r2_score(y_test, y_pred_test)
print(r2_test_lr)
metric2.append(r2_test_lr)

rss1_lr = np.sum(np.square(y_train - y_pred_train))
print(rss1_lr)
metric2.append(rss1_lr)

rss2_lr = np.sum(np.square(y_test - y_pred_test))
print(rss2_lr)
metric2.append(rss2_lr)

mse_train_lr = mean_squared_error(y_train, y_pred_train)
print(mse_train_lr)
metric2.append(mse_train_lr**0.5)

mse_test_lr = mean_squared_error(y_test, y_pred_test)
print(mse_test_lr)
metric2.append(mse_test_lr**0.5)
# -

# ### Top 20 features from ridge after 2x alpha

rank_ridge = ranking(np.abs(ridge.coef_), imp_col)
sorted(rank_ridge.items(), key=lambda item: item[1] , reverse=True)[:20]

# # Lasso

# +
lasso = Lasso()

# cross validation
model_cv = GridSearchCV(estimator = lasso, 
                        param_grid = params, 
                        scoring= 'neg_mean_absolute_error', 
                        cv = folds, 
                        return_train_score=True,
                        verbose = 1)            

model_cv.fit(X_train, y_train) 
# -

cv_result_l = pd.DataFrame(model_cv.cv_results_)
cv_result_l['param_alpha'] = cv_result_l['param_alpha'].astype('float32')
cv_result_l.head()


plt.figure(figsize=(16,8))
plt.plot(cv_result_l['param_alpha'],cv_result_l['mean_train_score'])
plt.plot(cv_result_l['param_alpha'],cv_result_l['mean_test_score'])
plt.xlabel('Alpha')
plt.xscale('log')
plt.ylabel('R2 Score')
plt.show()



# Printing the best hyperparameter alpha
print(model_cv.best_params_)

# +
#Fitting Lasso model for alpha = 0.0001 and printing coefficients which have been penalised

alpha =0.0001

lasso = Lasso(alpha=alpha)
        
lasso.fit(X_train, y_train) 

# +
# Lets calculate some metrics such as R2 score, RSS and RMSE

y_pred_train = lasso.predict(X_train)
y_pred_test = lasso.predict(X_test)

metric3 = []
r2_train_lr = r2_score(y_train, y_pred_train)
print(r2_train_lr)
metric3.append(r2_train_lr)

r2_test_lr = r2_score(y_test, y_pred_test)
print(r2_test_lr)
metric3.append(r2_test_lr)

rss1_lr = np.sum(np.square(y_train - y_pred_train))
print(rss1_lr)
metric3.append(rss1_lr)

rss2_lr = np.sum(np.square(y_test - y_pred_test))
print(rss2_lr)
metric3.append(rss2_lr)

mse_train_lr = mean_squared_error(y_train, y_pred_train)
print(mse_train_lr)
metric3.append(mse_train_lr**0.5)

mse_test_lr = mean_squared_error(y_test, y_pred_test)
print(mse_test_lr)
metric3.append(mse_test_lr**0.5)
# -

# ## Top 20 important features from lasso

rank_lasso = ranking(np.abs(lasso.coef_), imp_col)
top_lasso_features = sorted(rank_lasso.items(), key=lambda item: item[1] , reverse=True)[:20]
top_lasso_features



# ## According to Problem Statement - Part II What if we choose double value of alpha

# +
#Fitting lasso model for alpha = 0.0002 and printing coefficients which have been penalised

alpha2x =0.0002

lasso = Lasso(alpha=alpha2x)
        
lasso.fit(X_train, y_train) 

# +
# Lets calculate some metrics such as R2 score, RSS and RMSE

y_pred_train = lasso.predict(X_train)
y_pred_test = lasso.predict(X_test)

metric3 = []
r2_train_lr = r2_score(y_train, y_pred_train)
print(r2_train_lr)
metric3.append(r2_train_lr)

r2_test_lr = r2_score(y_test, y_pred_test)
print(r2_test_lr)
metric3.append(r2_test_lr)

rss1_lr = np.sum(np.square(y_train - y_pred_train))
print(rss1_lr)
metric3.append(rss1_lr)

rss2_lr = np.sum(np.square(y_test - y_pred_test))
print(rss2_lr)
metric3.append(rss2_lr)

mse_train_lr = mean_squared_error(y_train, y_pred_train)
print(mse_train_lr)
metric3.append(mse_train_lr**0.5)

mse_test_lr = mean_squared_error(y_test, y_pred_test)
print(mse_test_lr)
metric3.append(mse_test_lr**0.5)
# -

# ## Top 20 important features from lasso after 2x alpha

rank_lasso = ranking(np.abs(lasso.coef_), imp_col)
sorted(rank_lasso.items(), key=lambda item: item[1] , reverse=True)[:20]

# ## Analysis Ridge and Lasso

# +
# Creating a table which contain all the metrics

lr_table = {'Metric': ['R2 Score (Train)','R2 Score (Test)','RSS (Train)','RSS (Test)',
                       'MSE (Train)','MSE (Test)']
        }

lr_metric = pd.DataFrame(lr_table ,columns = ['Metric'] )

rg_metric = pd.Series(metric2, name = 'Ridge Regression')
ls_metric = pd.Series(metric3, name = 'Lasso Regression')

final_metric = pd.concat([lr_metric, rg_metric, ls_metric], axis = 1)

final_metric
# -

# # Final model : Lasso 
# ### Lasso regression would be a better option it would help in feature elimination and the model will be more robust.

# ## Lets observe the changes in the coefficients after regularization

betas = pd.DataFrame(index=X_train.columns)

betas.rows = X_train.columns

# +

betas['Ridge'] = ridge.coef_
betas['Lasso'] = lasso.coef_
# -

pd.set_option('display.max_rows', None)
betas.head(20)



# # Subjective Question 
# ## Question 3
# #### After building the model, you realised that the five most important predictor variables in the lasso model are not available in the incoming data. You will now have to create another model excluding the five most important predictor variables. Which are the five most important predictor variables now?
#

top_lasso_features

#     Let's drop the top 5 features from the model and build model again 

# ## Data Preparation

X_train_q3 = X_train.drop(['GrLivArea','Condition2_PosN','RoofMatl_WdShngl','OverallQual','Neighborhood_NoRidge'], axis = 1)
X_test_q3 = X_test.drop(['GrLivArea','Condition2_PosN','RoofMatl_WdShngl','OverallQual','Neighborhood_NoRidge'], axis = 1)

X_test_q3.shape

# +
lasso = Lasso()

# cross validation
model_cv = GridSearchCV(estimator = lasso, 
                        param_grid = params, 
                        scoring= 'neg_mean_absolute_error', 
                        cv = folds, 
                        return_train_score=True,
                        verbose = 1)            

model_cv.fit(X_train_q3, y_train) 
# -

# Printing the best hyperparameter alpha
print(model_cv.best_params_)

# +
#Fitting Ridge model for alpha = 1000 and printing coefficients which have been penalised

alpha =0.0001

lasso = Lasso(alpha=alpha)
        
lasso.fit(X_train_q3, y_train) 

# +
# Lets calculate some metrics such as R2 score, RSS and RMSE

y_pred_train = lasso.predict(X_train_q3)
y_pred_test = lasso.predict(X_test_q3)

metric3 = []
r2_train_lr = r2_score(y_train, y_pred_train)
print(r2_train_lr)
metric3.append(r2_train_lr)

r2_test_lr = r2_score(y_test, y_pred_test)
print(r2_test_lr)
metric3.append(r2_test_lr)

rss1_lr = np.sum(np.square(y_train - y_pred_train))
print(rss1_lr)
metric3.append(rss1_lr)

rss2_lr = np.sum(np.square(y_test - y_pred_test))
print(rss2_lr)
metric3.append(rss2_lr)

mse_train_lr = mean_squared_error(y_train, y_pred_train)
print(mse_train_lr)
metric3.append(mse_train_lr**0.5)

mse_test_lr = mean_squared_error(y_test, y_pred_test)
print(mse_test_lr)
metric3.append(mse_test_lr**0.5)
# -

rank_lasso = ranking(np.abs(lasso.coef_), imp_col)
top_lasso_q3 = sorted(rank_lasso.items(), key=lambda item: item[1] , reverse=True)[:20]
top_lasso_q3

# ### After dropping top 5 important features and then building the model again 
# ## top 5 features are
#
#     LowQualFinSF
#     1stFlrSF
#     Fireplaces
#     LotArea
#     Exterior2nd_Brk_Cmn





StarcoderdataJupyterScriptsDedupFiltered