# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Improving performance

import pandas as pd
import numpy as np
# %matplotlib inline
import matplotlib.pyplot as plt

# Load the data
df = pd.read_csv('../data/new_titanic_features.csv')

# Create Features and Labels
X = df[['Male', 'Family',
        'Pclass2_one', 'Pclass2_two', 'Pclass2_three',
        'Embarked_C', 'Embarked_Q', 'Embarked_S',
        'Age2', 'Fare3_Fare11to50', 'Fare3_Fare51+', 'Fare3_Fare<=10']]
y = df['Survived']


X.describe()

X.head(15)

from sklearn.model_selection import train_test_split

# Train test split
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=.2, random_state=0)

from sklearn.linear_model import LogisticRegression

model = LogisticRegression()
model.fit(X_train, y_train)
pred_train = model.predict(X_train)
pred_test = model.predict(X_test)

from sklearn.metrics import accuracy_score, confusion_matrix, classification_report

# Check the baseline accuracy against the actual distribution of survival rates:

y_train.value_counts() / len(y_train)

# We see that if we made a model that just predicted that everybody died, it would be correct 62% of the time.  So our scoring below (test accuracy of 72%) is somewhat better, but there is room for improvement.

print('Train Accuracy: {:0.3}'.format(accuracy_score(y_train, pred_train)))
print('Test Accuracy: {:0.3}'.format(accuracy_score(y_test, pred_test)))

confusion_matrix(y_test, pred_test)

print(classification_report(y_test, pred_test))

# ## Feature importances (wrong! see exercise 1)

# These are the coefficients that multiply the features in the model, plus the bias.
#
# Larger coefficient magnitude generally means more important feature.

model.coef_

# Can get bias too:
#

model.intercept_

df['Fare'].plot(kind='hist', bins=100)


df.groupby(['Pclass','Survived'])['PassengerId'].count()



coeffs = pd.Series(model.coef_.ravel(), index=X.columns)
coeffs

coeffs.plot(kind='barh')

# Have to be careful interpreting these as importance of the features-- have to look at the distributions and magnitudes of each feature.  The Age feature is not yet normalized and has a mean of 29, a max of 80, and a min of -5.  Have to normalize to get a better picture of the contributions of these different features.

# ## Cross Validation

from sklearn.model_selection import cross_val_score, ShuffleSplit

# +
cv = ShuffleSplit(n_splits=5, test_size=.4, random_state=0)
scores = cross_val_score(model, X, y, cv=cv)
#scores = cross_val_score(model, X, y, cv=3) #would be exclusive train/test corss validation

scores
# -

'Crossval score: %0.3f +/- %0.3f ' % (scores.mean(), scores.std())

# ## Learning curve

from sklearn.model_selection import learning_curve

tsz = np.linspace(0.1, 1, 10)
train_sizes, train_scores, test_scores = learning_curve(model, X, y, train_sizes=tsz)

fig = plt.figure()
plt.plot(train_sizes, train_scores.mean(axis=1), 'ro-', label="Train Scores")
plt.plot(train_sizes, test_scores.mean(axis=1), 'go-', label="Test Scores")
plt.title('Learning Curve: Logistic Regression')
plt.ylim((0.5, 1.0))
plt.legend()
plt.draw()
plt.show()

# Adding more data appears to decrease model performance, so we need to make changes to the model to make an improvement.

# ### Exercise 1
#
# Try rescaling the Age feature with [`preprocessing.StandardScaler`](http://scikit-learn.org/stable/modules/preprocessing.html) so that it will have comparable size to the other features.
#
# - Do the model prediction change?
# - Does the performance of the model change?
# - Do the feature importances change?
# - How can you explain what you've observed?

from sklearn.preprocessing import StandardScaler

X_train['Age2'].plot(kind='hist', bins = 30)

scaler = StandardScaler()
scaler.fit(X_train[['Age2']])

# +
X_train_scaled = X_train.copy()
X_test_scaled = X_test.copy()


X_train_scaled['Age2'] = scaler.transform(X_train[['Age2']])
X_test_scaled['Age2'] = scaler.transform(X_test[['Age2']])

# -

X_train_scaled['Age2'].plot(kind='hist', bins = 30)

model = LogisticRegression()
model.fit(X_train_scaled, y_train)
pred_train = model.predict(X_train_scaled)
pred_test = model.predict(X_test_scaled)

# +
##not finished, return to check solution
# -





# ### Exercise 2
#
# Experiment with another classifier for example `DecisionTreeClassifier`, `RandomForestClassifier`,  `SVC`, `MLPClassifier`, `SGDClassifier` or any other classifier of choice you can find here: http://scikit-learn.org/stable/auto_examples/classification/plot_classifier_comparison.html. 
#
# - Train the model on both the scaled data and on the unscaled data
# - Compare the score for the scaled and unscaled data
# - how can you get the features importances for tree based models? Check [here](http://scikit-learn.org/stable/auto_examples/ensemble/plot_forest_importances.html) for some help.
# - Which classifiers are impacted by the age rescale? Why?

# +
from sklearn.ensemble import RandomForestClassifier

model = RandomForestClassifier()
model.fit(X_train, y_train)
pred_train = model.predict(X_train)
pred_test = model.predict(X_test)

coeffs = pd.Series(model.feature_importances_, index=X.columns)
coeffs.plot(kind='barh')

print('Train Accuracy: {:0.3}'.format(accuracy_score(y_train, pred_train)))
print('Test Accuracy: {:0.3}'.format(accuracy_score(y_test, pred_test)))

# -







# ### Exercise 3
#
# Pick your preferred classifier from Exercise 2 and search for the best hyperparameters. You can read about hyperparameter search [here](http://scikit-learn.org/stable/modules/grid_search.html)
#
# - Decide the range of hyperparameters you intend to explore
# - Try using [`GridSearchCV`](http://scikit-learn.org/stable/modules/generated/sklearn.model_selection.GridSearchCV.html#sklearn.model_selection.GridSearchCV) to perform brute force search
# - Try using [`RandomizedSearchCV`](http://scikit-learn.org/stable/modules/generated/sklearn.model_selection.RandomizedSearchCV.html#sklearn.model_selection.RandomizedSearchCV) for a random search
# - Once you've chosen the best classifier and the best hyperparameter set, redo the learning curve.
# Do you need more data or a better model?



StarcoderdataJupyterScriptsDedupFiltered