# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
# %matplotlib inline

data = pd.read_csv('train.csv')
data.head()

fig,ax = plt.subplots()
fig.set_size_inches(10,15)
sns.heatmap(data.isnull(),cbar=False,cmap='magma')

# Looks like we're missing two data point from Embarked some data from Age and a lot from Cabin
#
# Solution:
# 1. Drop the two in Embarked
# 2. Drop cabin
# 3. Try to use linear regression to predict the age for the null ones
#

data = data[~data.Embarked.isnull()]
data.drop('Cabin',axis=1,inplace=True)
data.head()

# I'm gonna make dummy variables for Pclass which is the fare class and Sex. Embarked will get the same treatment as well as it identifies the port at which the passenger embarked. I'll drop the first row of dummy variables because it can be inferred from the others

pclass = pd.get_dummies(data.Pclass,drop_first=True)
sex = pd.get_dummies(data.Sex,drop_first=True)
embarked = pd.get_dummies(data.Embarked,drop_first=True)
data = pd.concat([data,pclass,sex,embarked],axis=1)
del pclass
del sex
del embarked
data.head()

# Sex, Pclass and Embarked are now redundant so I'll drop them. I'm also gonna drop Name and Ticket as they need further feature engineering to be useful. PasengerId gives no useful information so that'll be dropped as well

data.drop(['Sex','Pclass','Embarked','Name','Ticket','PassengerId'],axis=1,inplace=True)
data.head()

# Now I'll split the data with non-null age values into a train and test set and try to learn the relationship between age and everything but Survived

survived = data.Survived
data.drop('Survived',inplace=True,axis=1)
data_good = data[~data.Age.isnull()]
data_good.head()

from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from regressors import stats
X_train, X_test, y_train, y_test = train_test_split(data_good.drop('Age',axis=1), data_good.Age, test_size=0.2, random_state=101)
X_train

lm = LinearRegression()
lm.fit(X_train,y_train)
coeff_df = pd.DataFrame(lm.coef_,data_good.drop('Age',axis=1).columns,columns=['Coefficient'])
coeff_df

from sklearn import metrics
predictions = lm.predict(X_test)
print('MAE:', metrics.mean_absolute_error(y_test, predictions))
print('MSE:', metrics.mean_squared_error(y_test, predictions))
print('RMSE:', np.sqrt(metrics.mean_squared_error(y_test, predictions)))

# So the linear regression has a MAE of 9.8 years. I'm gonna fill the empty age values with the predictions from the linear regression model

data_notgood = data[data.Age.isnull()]
data_notgood.head()

lm.predict(data_notgood.drop('Age',axis=1))

# It appears that the model is predicting negative values which are obviously wrong. This happens because the model does not respect the 0 bound. To address this, I'm gonna retrain the model using the natural log of the age and predict that.

lm = LinearRegression()
lm.fit(X_train,np.log(y_train))
coeff_df = pd.DataFrame(lm.coef_,data_good.drop('Age',axis=1).columns,columns=['Coefficient'])
coeff_df

from sklearn import metrics
predictions = np.exp(lm.predict(X_test))
print('MAE:', metrics.mean_absolute_error(y_test, predictions))
print('MSE:', metrics.mean_squared_error(y_test, predictions))
print('RMSE:', np.sqrt(metrics.mean_squared_error(y_test, predictions)))

data_notgood['Age'] = np.exp(lm.predict(data_notgood.drop('Age',axis=1)))

data.head()

data = pd.concat([data_good,data_notgood],axis=0)
data.sort_index(inplace=True)
data.head()

fig,ax = plt.subplots()
fig.set_size_inches(10,15)
sns.heatmap(data.isnull(),cbar=False,cmap='magma')

# Now all null values have either been dropped or replaced by an "educated" guess, we can continue with training the logistic regression

from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score
from sklearn.metrics import classification_report
lrm = LogisticRegression(solver='lbfgs',max_iter=1000)
lrm = RandomForestClassifier(n_estimators=100)
X_train, X_test, y_train, y_test = train_test_split(data, survived, test_size=0.2, random_state=101)
lrm.fit(X_train,y_train)
lrm.score(X_test,y_test)
predictions = lrm.predict(X_test)
print(classification_report(y_test,predictions))

data

# Now we have a baseline in place, lets try to predict from the test dataset. For this, the same preprocessing steps will need to be applied to the test set. Lets examine this dataset as well

data_test = pd.read_csv('test.csv')
passenger_id = data_test.PassengerId
data_test.info()

# It seems there is one missing value for the Fare. Lets see if we can quickly fill that in by analyzing the fare's relationship to the Pclass

sns.set_style('darkgrid')
sns.boxplot(x="Pclass",y="Fare",data=data_test,hue='Sex',showfliers=False)

# So it seems the sex and cabin class can be a good identifier for fare. I'll just replace the missing fare with the mean of the fare for that class/sex combination

data_test[data_test.Fare.isnull()]

data_test['Fare'].fillna(data_test[(data_test.Pclass == 3) & (data_test.Sex == "male")].Fare.mean(),inplace=True)

data_test.drop('Cabin',axis=1,inplace=True)
pclass = pd.get_dummies(data_test.Pclass,drop_first=True)
sex = pd.get_dummies(data_test.Sex,drop_first=True)
embarked = pd.get_dummies(data_test.Embarked,drop_first=True)
data_test = pd.concat([data_test,pclass,sex,embarked],axis=1)
data_test.drop(['Sex','Pclass','Embarked','Name','Ticket','PassengerId'],axis=1,inplace=True)
data_test.head()

data_test_good = data_test[~data_test.Age.isnull()]
data_test_notgood = data_test[data_test.Age.isnull()]
data_test_notgood['Age'] = np.exp(lm.predict(data_test_notgood.drop('Age',axis=1)))
data_test = pd.concat([data_test_good,data_test_notgood],axis=0)
data_test.sort_index(inplace=True)
data_test.head()

predictions_test = lrm.predict(data_test)

out = pd.DataFrame(data = {"PassengerId": passenger_id, "Survived": predictions_test})
out.to_csv('submission.csv',index=False)

StarcoderdataJupyterScriptsDedupFiltered