<jupyter_start><jupyter_text>Question-1 - Linear reg from scratch_grdient descent<jupyter_code># loading required libraries & data 
import numpy as np 
import pandas as pd 
import matplotlib.pyplot as plt
import seaborn as sns

import warnings
warnings.filterwarnings('ignore')<jupyter_output><empty_output><jupyter_text>**Acknowledgements**: I've primariliy used the material from [Andrew Ng's Coursera course][1] for this, but have also been helped by [this article][2] and [this one][3]. I used some code for the animation from [this kernel][4].  [1]: https://www.coursera.org/learn/machine-learning  [2]: http://tillbergmann.com/blog/python-gradient-descent.html  [3]: http://aimotion.blogspot.co.uk/2011/10/machine-learning-with-python-linear.html  [4]: https://www.kaggle.com/ronaldtroncoso20/d/START-UMD/gtd/global-terrorism-trends-animation<jupyter_code>data = pd.read_csv("train.csv")
#Grab the relevant data, scale the predictor variable, and add a column of 1s for the gradient descent...
x = data['GrLivArea']
y = data['SalePrice']

x = (x - x.mean()) / x.std()
x = np.c_[np.ones(x.shape[0]), x]
#GRADIENT DESCENT

alpha = 0.01 #Step size
iterations = 2000 #No. of iterations
m = y.size #No. of data points
np.random.seed(123) #Set the seed
theta = np.random.rand(2) #Pick some random values to start with

def gradient_descent(x, y, theta, iterations, alpha):
    past_costs = []
    past_thetas = [theta]
    for i in range(iterations):
        prediction = np.dot(x, theta)
        error = prediction - y
        cost = 1/(2*m) * np.dot(error.T, error)
        past_costs.append(cost)
        theta = theta - (alpha * (1/m) * np.dot(x.T, error))
        past_thetas.append(theta)
        
    return past_thetas, past_costs

#Pass the relevant variables to the function and get the new values
past_thetas, past_costs = gradient_descent(x, y, theta, iterations, alpha)
theta = past_thetas[-1]

# Results...
print("Gradient Descent: {:.2f}, {:.2f}".format(theta[0], theta[1]))<jupyter_output>Gradient Descent: 180921.20, 56294.90<jupyter_text>Questin-2-Kaggle Competition: House Prices Prediction - Advanced Regression Techniques  kaggle competition link: https://www.kaggle.com/c/house-prices-advanced-regressiontechniques<jupyter_code># reading the dataset

train =pd.read_csv('train.csv')
test = pd.read_csv('test.csv')

# checking the shapes of the train and test datasets
print("dimensions of train dataset: ", train.shape)
print("dimensions of test dataset: ", test.shape)
train.head()
train.drop("Id", axis = 1, inplace = True)
test.drop("Id", axis = 1, inplace = True)

train1 = train.copy()
train1 = train1.drop(train1[(train1['GarageArea']>1200) & (train1['SalePrice']<300000)].index)
train1 = train1.drop(train1[(train1['GrLivArea']>4000) & (train1['SalePrice']<300000)].index)
train1 = train1.drop(train1[(train1['TotalBsmtSF']>5000)].index)

# Split X and y (in train dataset)
X = train1.drop('SalePrice', axis=1)
y = train1['SalePrice'].to_frame()

# Add variable
X['train'] = 1
test['train'] = 0

# Combining train and test for data cleaning 
df = pd.concat([test, X])
#  duplicates
print('Number of Duplicates:', len(df[df.duplicated()]))

# missing values
print('Number of Missing Values:', df.isnull().sum().sum())<jupyter_output><empty_output><jupyter_text>Balancing missing data<jupyter_code>print('Missing Values per Column:')
df.isnull().sum().sort_values(ascending=False).head(25)
df['PoolQC'] = df['PoolQC'].fillna('None')
df['MiscFeature'] = df['MiscFeature'].fillna('None')
df['Alley'] = df['Alley'].fillna('None')
df['Fence'] = df['Fence'].fillna('None')
df['FireplaceQu'] = df['FireplaceQu'].fillna('None')
df['LotFrontage'] = df.groupby('Neighborhood')['LotFrontage'].transform(lambda i: i.fillna(i.median()))
# Let's take a look at the "Garage" features
garage_cols = [col for col in df if col.startswith('Garage')]
df[garage_cols]
# For the numerical features:
for i in df[garage_cols].select_dtypes(exclude='object').columns:
    df[i] = df[i].fillna(0)

# For the categorical features:
for i in df[garage_cols].select_dtypes(include='object').columns:
    df[i] = df[i].fillna('None')
bsmt_cols = [col for col in df if col.startswith('Bsmt')]

# For the numerical features:
for i in df[bsmt_cols].select_dtypes(exclude='object').columns:
    df[i] = df[i].fillna(0)

# For the categorical features:
for i in df[bsmt_cols].select_dtypes(include='object').columns:
    df[i] = df[i].fillna('None')

mas_cols = [col for col in df if col.startswith('Mas')]

# For the numerical features:
for i in df[mas_cols].select_dtypes(exclude='object').columns:
    df[i] = df[i].fillna(0)

# For the categorical features:
for i in df[mas_cols].select_dtypes(include='object').columns:
    df[i] = df[i].fillna('None')
df['MSZoning'] = df.groupby('Neighborhood')['MSZoning'].transform(lambda i: i.fillna(i.value_counts().index[0]))
# replace missing values for mode of each column
df = df.fillna(df.mode().iloc[0])<jupyter_output><empty_output><jupyter_text>Transforming some numerical categories into categorical<jupyter_code>df['MSSubClass'] = df['MSSubClass'].astype(str)
df['MoSold'] = df['MoSold'].astype(str)           # months is always categorical
df['YrSold'] = df['YrSold'].astype(str)
# adding some features to increase the accuracy of prediction 
df['Total_House_SF'] = df['TotalBsmtSF'] + df['1stFlrSF'] + df['2ndFlrSF']
df['Total_Home_Quality'] = (df['OverallQual'] + df['OverallCond'])/2
df['Total_Bathrooms'] = (df['FullBath'] + (0.5 * df['HalfBath']) + df['BsmtFullBath'] + (0.5 * df['BsmtHalfBath']))
# selecting the features that have a skew higher than 0.5.

numeric_cols = df.select_dtypes(exclude='object').columns

skew_limit = 0.5
skew_vals = df[numeric_cols].skew()

skew_cols = (skew_vals
             .sort_values(ascending=False)
             .to_frame()
             .rename(columns={0:'Skew'})
             .query('abs(Skew) > {0}'.format(skew_limit)))

skew_cols<jupyter_output><empty_output><jupyter_text>Encoding categorical features<jupyter_code>categ_cols = df.dtypes[df.dtypes == np.object]        # filtering by categorical variables
categ_cols = categ_cols.index.tolist()                # list of categorical fields

df_enc = pd.get_dummies(df, columns=categ_cols, drop_first=True)   # One hot encoding
X = df_enc[df_enc['train']==1]
test = df_enc[df_enc['train']==0]
X.drop(['train'], axis=1, inplace=True)
test.drop(['train'], axis=1, inplace=True)<jupyter_output><empty_output><jupyter_text>Train-test split<jupyter_code>from sklearn.linear_model import Ridge, RidgeCV, Lasso, LassoCV
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=12345)
def rmse(ytrue, ypredicted):
    return np.sqrt(mean_squared_error(ytrue, ypredicted))<jupyter_output><empty_output><jupyter_text>Lasso + cross validation<jupyter_code>lasso = Lasso(max_iter = 100000, normalize = True)

lassocv = LassoCV(alphas = None, cv = 10, max_iter = 100000, normalize = True)
lassocv.fit(X_train, y_train)

lasso.set_params(alpha=lassocv.alpha_)
lasso.fit(X_train, y_train)

print('The Lasso I:')
print("Alpha =", lassocv.alpha_)
print("RMSE =", rmse(y_test, lasso.predict(X_test)))<jupyter_output>The Lasso I:
Alpha = 23.00420091860413
RMSE = 22141.967752032626<jupyter_text>Ridge + cross validtion<jupyter_code>alphas = np.geomspace(1e-9, 5, num=100)

ridgecv = RidgeCV(alphas = alphas, scoring = 'neg_mean_squared_error', normalize = True)
ridgecv.fit(X_train, y_train)

ridge = Ridge(alpha = ridgecv.alpha_, normalize = True)
ridge.fit(X_train, y_train)

print('Ridge Regression:')
print("Alpha =", ridgecv.alpha_)
print("RMSE =", rmse(y_test, ridge.predict(X_test)))<jupyter_output>Ridge Regression:
Alpha = 0.21251935471767855
RMSE = 22812.490982144296
StarcoderdataJupyterStructuredCleanDedup