# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + _cell_guid="b1076dfc-b9ad-4769-8c92-a6c4dae69d19" _uuid="8f2839f25d086af736a60e9eeb907d3b93b6e0e5" papermill={"duration": 1.123465, "end_time": "2021-07-26T05:28:47.868961", "exception": false, "start_time": "2021-07-26T05:28:46.745496", "status": "completed"} tags=[]
# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_squared_error
from sklearn.metrics import accuracy_score
from sklearn.metrics import roc_auc_score
import matplotlib.pyplot as pt
import seaborn as sns
from sklearn.model_selection import train_test_split
pd.set_option('display.max_columns', None)
# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session

# + [markdown] papermill={"duration": 0.022244, "end_time": "2021-07-26T05:28:47.915261", "exception": false, "start_time": "2021-07-26T05:28:47.893017", "status": "completed"} tags=[]
# # Input of training data from input folder 
# Links - https://www.kaggle.com/azeembootwala/titanic

# + papermill={"duration": 0.078653, "end_time": "2021-07-26T05:28:48.016371", "exception": false, "start_time": "2021-07-26T05:28:47.937718", "status": "completed"} tags=[]
input_ads = pd.read_csv('../input/titanic/train_data.csv')
input_ads.drop(columns=['Unnamed: 0','Title_1','Title_2','Title_3','Title_4'],inplace=True) #Dropping un-necessary columns
#-----------------------------------------------------------------
print(input_ads.shape)
input_ads.head()

# + [markdown] papermill={"duration": 0.023364, "end_time": "2021-07-26T05:28:48.063177", "exception": false, "start_time": "2021-07-26T05:28:48.039813", "status": "completed"} tags=[]
# # Null Check

# + papermill={"duration": 0.044028, "end_time": "2021-07-26T05:28:48.131003", "exception": false, "start_time": "2021-07-26T05:28:48.086975", "status": "completed"} tags=[]
pd.DataFrame(input_ads.isnull().sum()).T

# + [markdown] papermill={"duration": 0.023811, "end_time": "2021-07-26T05:28:48.180166", "exception": false, "start_time": "2021-07-26T05:28:48.156355", "status": "completed"} tags=[]
# # Describe of the whole data

# + papermill={"duration": 0.082647, "end_time": "2021-07-26T05:28:48.287308", "exception": false, "start_time": "2021-07-26T05:28:48.204661", "status": "completed"} tags=[]
input_ads.describe()

# + [markdown] papermill={"duration": 0.024066, "end_time": "2021-07-26T05:28:48.335969", "exception": false, "start_time": "2021-07-26T05:28:48.311903", "status": "completed"} tags=[]
# ### Note - The data is already standardized since it is imported from a pre-processed public dataset

# + papermill={"duration": 0.036644, "end_time": "2021-07-26T05:28:48.396994", "exception": false, "start_time": "2021-07-26T05:28:48.360350", "status": "completed"} tags=[]
#Total survived vs not-survived split in the training data
input_ads['Survived'].value_counts()

# + [markdown] papermill={"duration": 0.024375, "end_time": "2021-07-26T05:28:48.446310", "exception": false, "start_time": "2021-07-26T05:28:48.421935", "status": "completed"} tags=[]
# # Data Splitting & Pre-Processing 

# + papermill={"duration": 0.05372, "end_time": "2021-07-26T05:28:48.524769", "exception": false, "start_time": "2021-07-26T05:28:48.471049", "status": "completed"} tags=[]
target = 'Survived' #To predict

#--------------------------------------------------------------------------------
#Splitting into X & Y datasets (supervised training)
X = input_ads[[cols for cols in list(input_ads.columns) if target not in cols]]
y = input_ads[target]

#--------------------------------------------------------------------------------
#Since test data is already placed in the input folder separately, we will just import it
test_ads = pd.read_csv('../input/titanic/test_data.csv')
test_ads.drop(columns=['Unnamed: 0','Title_1','Title_2','Title_3','Title_4'],inplace=True) #Dropping un-necessary columns

#Splitting into X & Y datasets (supervised training)
X_test = test_ads[[cols for cols in list(test_ads.columns) if target not in cols]]
y_test = test_ads[target]

print('Train % of total data:',100 * X.shape[0]/(X.shape[0] + X_test.shape[0]))
#--------------------------------------------------------------------------------
#Manipulation of datasets for convenience and consistency
X_arr = np.array(X)
X_test_arr = np.array(X_test)

y_arr = np.array(y).reshape(X_arr.shape[0],1)
y_test_arr = np.array(y_test).reshape(X_test_arr.shape[0],1)

#--------------------------------------------------------------------------------
#Basic Summary
print(X_arr.shape)
print(X_test_arr.shape)
print(y_arr.shape)


# + [markdown] papermill={"duration": 0.02458, "end_time": "2021-07-26T05:28:48.574178", "exception": false, "start_time": "2021-07-26T05:28:48.549598", "status": "completed"} tags=[]
# # Logistic Regression from scratch

# + [markdown] papermill={"duration": 0.02451, "end_time": "2021-07-26T05:28:48.623562", "exception": false, "start_time": "2021-07-26T05:28:48.599052", "status": "completed"} tags=[]
# ### Defining fwd prop UDF, Cost function UDF & initiating weights and intercepts

# + papermill={"duration": 0.045569, "end_time": "2021-07-26T05:28:48.694159", "exception": false, "start_time": "2021-07-26T05:28:48.648590", "status": "completed"} tags=[]
#Sigmoid function for the forward propagation as well as backward propagation
def sigmoid(arr):
    
    sig = 1/(1 + np.exp(-arr))
    
    return sig

#Fn for forward propagation of the model (to caculate the predictions)
#--------------------------------------------------------------------------
def fwd_prop(X_arr,w,b):
    
    a = np.dot(X_arr,w) + b
    sig_a = sigmoid(a)
    #print('Shape of a:',a.shape)
    
    return sig_a

#Fn to calculate cost for logistic regression
#--------------------------------------------------------------------------------------------------
def cost_fn(y_true,y_pred,n_examples,reg_alpha,reg_type,w_):
    
    #Applying regularizations
    if reg_type=='L1':
        reg = np.sum(abs(w_))
    elif reg_type=='L2':
        reg = 0.5 * np.sum(np.square(w_))
    
    cost = (-1/n_examples) * np.sum((y_true * np.log(y_pred)) + ((1-y_true) * np.log(1 - y_pred))) + (reg_alpha*reg)
    
    return cost    

#Fn to convert probabilities into class 0 or 1 based on threshold
#--------------------------------------------------------------------------
def prob_to_class(arr,threshold):
    
    mask = arr>threshold
    #print(mask)
    arr_class = mask.astype(int)
    
    return arr_class


#Initiating the weight and intercept vectors with appropriate dimensions
#--------------------------------------------------------------------------
np.random.seed(100) #Setting seed for consistency in case of random number generation 

#Weights
#----------------------------------
w = np.zeros((X.shape[1],1))
print(X_arr.shape[1])
print(w.shape)
print(w)

#Intercept
#----------------------------------
b = np.zeros(1)
b

#Testing the forward propagation function
#----------------------------------
a = fwd_prop(X_arr,w,b)
a.shape


# + [markdown] papermill={"duration": 0.02577, "end_time": "2021-07-26T05:28:48.746278", "exception": false, "start_time": "2021-07-26T05:28:48.720508", "status": "completed"} tags=[]
# ### UDF for batch_gradient_descent
# #### 1. If batch_size=1, it becomes stochastic gradient descent

# + papermill={"duration": 0.045224, "end_time": "2021-07-26T05:28:48.817395", "exception": false, "start_time": "2021-07-26T05:28:48.772171", "status": "completed"} tags=[]
def batch_gradient_descent(y_arr_overall,X_arr_overall,w_,b_,n_iters=10,lr=0.01,batch_size=1,reg_alpha=1,reg_type='L2'):
    
    print('Total training rows :',X_arr_overall.shape[0])
    
    #----------------------------------------------------------------------------------------
    #Creating x-y batches according to the provided batch_size
    
    n_batches = X.shape[0]//batch_size
    print('Total Batches to create in each epoch/iter :',n_batches)
    
    batches_x = np.array_split(X_arr_overall,n_batches)
    print('Total Batches of X:',len(batches_x))

    batches_y = np.array_split(y_arr,n_batches)
    print('Total Batches of y:',len(batches_y))
    
    cost_history = [] #Cache for cost function o/p at necessary intervals for plotting later

    #----------------------------------------------------------------------------------------
    for i in range(n_iters): #Total iterations/epochs to train on
        
        if i%1000==0:
            print('#-------------------- Epoch number :',i,'--------------------#')
        
        for j in range(len(batches_x)): #For each batch created for each epoch/iter
            
            #print('Batch No :',j)
            
            X_arr_ = batches_x[j]
            y_arr_ = batches_y[j]

            n_examples = X_arr_.shape[0]
            #print(n_examples)
            #----------------------------------------------------------------------------------------
            #Forward propagation of the model - calculation of the model prediction
            a_temp = fwd_prop(X_arr_,w_,b_)

            cost = cost_fn(y_arr_,a_temp,n_examples,reg_alpha,reg_type,w_)
            
            if cost == np.inf:
                print('---- Inf encountered due to exploding gradients ----')
                return w_,b_,cost_history

            #----------------------------------------------------------------------------------------
            
            if reg_type=='L1':
                
                reg_derivative = np.divide(w_, abs(w_), out=np.zeros_like(w_), where=abs(w_)!=0)
                reg_derivative = np.where(reg_derivative==np.inf,0,reg_derivative)
                
            elif reg_type=='L2':
                
                reg_derivative = w_
            
            #Calculating the gradients for the current batch
            dz = (1/n_examples) * ((a_temp-y_arr_))  #Derivative of Loss fn 'L'(binary crossentropy) wrt z = wx+b
            
            dw = np.dot(X_arr_.T,dz) + ((1/n_examples)*(reg_alpha * reg_derivative)) #Derivative of w (weights) wrt 'L' (Applying chain rule of differentiation)
            db = (1/n_examples) * np.sum(dz) #Derivative of b (intercept) wrt 'L' (Applying chain rule of differentiation)
            
            #Updating the weight and the intercept
            w_ = w_ - (lr * dw)
            b_ = b_ - (lr * db)
        
        #Updating cost into the cache
        cost_history = cost_history + [cost]
        
        #-------------------------------------------------
        #Progress at regular intervals
        if (i%5000==0):
            print(i,': Cost ------->',cost)
            
            f_train_a = fwd_prop(X_arr_overall,w_,b_) #Results on whole training data after every 5k epochs
            #print(f_train_a.shape)
            
            f_train_a = prob_to_class(arr=f_train_a,threshold=0.5)
            print(f_train_a.shape)
            #print(y_arr_overall.shape)
            
            print('ROC AUC of training set :',roc_auc_score(y_arr_overall,f_train_a))
            print('Accuracy of training set :',accuracy_score(y_arr_overall,f_train_a))
        
    return w_,b_,cost_history


# + [markdown] papermill={"duration": 0.025386, "end_time": "2021-07-26T05:28:48.868578", "exception": false, "start_time": "2021-07-26T05:28:48.843192", "status": "completed"} tags=[]
# ### Training the logistic regression model

# + papermill={"duration": 32.939846, "end_time": "2021-07-26T05:29:21.834557", "exception": false, "start_time": "2021-07-26T05:28:48.894711", "status": "completed"} tags=[]
#Training the model on the training data with the below training specs
#-----------------------------------------------------------------------------------------------------------------------
epochs = 20001
learning_rate=0.00006
batch_size_=50

#-----------------------------------------------------------------------------------------------------------------------
w_final,b_final,cost_history = batch_gradient_descent(y_arr_overall=y_arr,       #Train y array 
                                                      X_arr_overall=X_arr,       #Train X array
                                                      w_=w,                      #Passing zero initiated weight vector
                                                      b_=b,                      #Passing zero initiaed intercept vector
                                                      n_iters=epochs,            #Total epochs/iters for Gradient Descent
                                                      lr=learning_rate,          #Learning rate for Gradient Descent
                                                      batch_size=batch_size_,    #Batch size for Gradient Descent (1 for SGD)
                                                      reg_alpha=0.05,            #Regularization factor
                                                      reg_type='L1')             #Regularization Type
                                                      

# + [markdown] papermill={"duration": 0.031872, "end_time": "2021-07-26T05:29:21.898525", "exception": false, "start_time": "2021-07-26T05:29:21.866653", "status": "completed"} tags=[]
# ### Plotting cost over epochs (Should have a sharp decrease)

# + papermill={"duration": 1.377651, "end_time": "2021-07-26T05:29:23.308133", "exception": false, "start_time": "2021-07-26T05:29:21.930482", "status": "completed"} tags=[]
#Cost plot over epochs (1 value at end of each epoch) - over the last batch
sns.set_style('darkgrid')
ax = sns.lineplot(x=list(range(0,epochs)),y=cost_history)
ax.set(xlabel='No of epochs',ylabel='Cost',title='Cost vs Epochs - Logistic Regression')


# + [markdown] papermill={"duration": 0.033019, "end_time": "2021-07-26T05:29:23.375003", "exception": false, "start_time": "2021-07-26T05:29:23.341984", "status": "completed"} tags=[]
# ### UDF for predicting

# + papermill={"duration": 0.043439, "end_time": "2021-07-26T05:29:23.451733", "exception": false, "start_time": "2021-07-26T05:29:23.408294", "status": "completed"} tags=[]
def predict(w_,b_,test_x,test_y):
    
    print("Testing on :",test_x.shape[0],'rows')
    
    a_temp = fwd_prop(test_x,w_,b_) #Using the weights(w_) and bias(b_) vectors derived from training 
    a_temp = prob_to_class(arr=a_temp,threshold=0.5)
    
    print('Shape of prediction :',a_temp.shape)
    
    print('ROC AUC of test set :',roc_auc_score(test_y,a_temp))
    print('Accuracy of test set :',accuracy_score(test_y,a_temp))
    
    print(a_temp[0:3])
    
    return a_temp



# + [markdown] papermill={"duration": 0.033609, "end_time": "2021-07-26T05:29:23.519425", "exception": false, "start_time": "2021-07-26T05:29:23.485816", "status": "completed"} tags=[]
# # Predictions from the manual created linear regression model

# + papermill={"duration": 0.046624, "end_time": "2021-07-26T05:29:23.599635", "exception": false, "start_time": "2021-07-26T05:29:23.553011", "status": "completed"} tags=[]
predictions_ = predict(w_final,b_final,X_test_arr,y_test_arr)

# + [markdown] papermill={"duration": 0.033442, "end_time": "2021-07-26T05:29:23.666971", "exception": false, "start_time": "2021-07-26T05:29:23.633529", "status": "completed"} tags=[]
# # Linear Regression from sklearn as benchmark

# + papermill={"duration": 0.833429, "end_time": "2021-07-26T05:29:24.534275", "exception": false, "start_time": "2021-07-26T05:29:23.700846", "status": "completed"} tags=[]
from sklearn.linear_model import LogisticRegression

#---------------------------------------------------------------------------------------
log_reg = LogisticRegression(penalty='l2',random_state=100,solver='sag',max_iter=20001,tol=1e-4,C=20)
log_reg.fit(X_arr,y_arr)

prediction_sklearn = log_reg.predict(X_test_arr)

#---------------------------------------------------------------------------------------
print('ROC AUC of test set :',roc_auc_score(y_test_arr,prediction_sklearn))
print('Accuracy of test set :',accuracy_score(y_test_arr,prediction_sklearn))

# + [markdown] papermill={"duration": 0.033841, "end_time": "2021-07-26T05:29:24.603395", "exception": false, "start_time": "2021-07-26T05:29:24.569554", "status": "completed"} tags=[]
# # SGD Classifier
# - (Logistic regression with normal stochastic gradient descent) - Directly comparable with manual implementation

# + papermill={"duration": 0.059878, "end_time": "2021-07-26T05:29:24.697642", "exception": false, "start_time": "2021-07-26T05:29:24.637764", "status": "completed"} tags=[]
from sklearn.linear_model import SGDClassifier

#---------------------------------------------------------------------------------------
log_reg_sgd = SGDClassifier(loss='log',penalty='l2',alpha=0.05,random_state=100,epsilon=learning_rate,max_iter=epochs,tol=1e-10)
log_reg_sgd.fit(X_arr,y_arr)

prediction_sklearn_sgd = log_reg_sgd.predict(X_test_arr)

#---------------------------------------------------------------------------------------
print('ROC AUC of test set :',roc_auc_score(y_test_arr,prediction_sklearn_sgd))
print('Accuracy of test set :',accuracy_score(y_test_arr,prediction_sklearn_sgd))

# + [markdown] papermill={"duration": 0.035107, "end_time": "2021-07-26T05:29:24.768326", "exception": false, "start_time": "2021-07-26T05:29:24.733219", "status": "completed"} tags=[]
# # Comparison to SGD Classifier (Because, sklearn logistic regression has other optimization techniques, its not directly comparable) 

# + [markdown] papermill={"duration": 0.034991, "end_time": "2021-07-26T05:29:24.838288", "exception": false, "start_time": "2021-07-26T05:29:24.803297", "status": "completed"} tags=[]
# ### Percent deviation in the weights (with respect to manual logistic regression weights)

# + papermill={"duration": 0.045572, "end_time": "2021-07-26T05:29:24.919133", "exception": false, "start_time": "2021-07-26T05:29:24.873561", "status": "completed"} tags=[]
100 * (w_final-log_reg_sgd.coef_.ravel().reshape(11,1))/w_final

# + [markdown] papermill={"duration": 0.035106, "end_time": "2021-07-26T05:29:24.989741", "exception": false, "start_time": "2021-07-26T05:29:24.954635", "status": "completed"} tags=[]
# ## Insights : 
# 1. Though the weights are very different as we see above, the models are predicting similarly.
# 2. This indicates that there are multiple solutions possible for the data in hand

# + [markdown] papermill={"duration": 0.035422, "end_time": "2021-07-26T05:29:25.061409", "exception": false, "start_time": "2021-07-26T05:29:25.025987", "status": "completed"} tags=[]
# # END

# + papermill={"duration": 0.036186, "end_time": "2021-07-26T05:29:25.133339", "exception": false, "start_time": "2021-07-26T05:29:25.097153", "status": "completed"} tags=[]


StarcoderdataJupyterScriptsDedupFiltered