# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] colab_type="text" id="PC9RfopIWrc9"
#  # Data Science Unit 2 Sprint Challenge 4 — Model Validation

# + [markdown] colab_type="text" id="UV7ArLFQN84W"
# Follow the instructions for each numbered part to earn a score of 2. See the bottom of the notebook for a list of ways you can earn a score of 3.

# + [markdown] colab_type="text" id="bAZcbTtiUlkI"
# ## Predicting Blood Donations
#
# Our dataset is from a mobile blood donation vehicle in Taiwan. The Blood Transfusion Service Center drives to different universities and collects blood as part of a blood drive.
#
# **The goal is to predict the last column, whether the donor made a donation in March 2007**, using information about each donor's history. We'll measure success **using recall score** as the model evaluation metric.
#
# Good data-driven systems for tracking and predicting donations and supply needs can improve the entire supply chain, making sure that more patients get the blood transfusions they need.
#
# #### Run this cell to load the data:

# + colab={} colab_type="code" id="WHY4eFBp4B5k"
# #!pip install seaborn --upgrade
import pandas as pd
import numpy as np
import numpy as np
import pandas as pd
from sklearn.feature_selection import f_classif, SelectKBest
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, recall_score, f1_score
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.dummy import DummyClassifier
from sklearn.ensemble import RandomForestClassifier
import warnings
import time
warnings.filterwarnings('ignore')

import seaborn as sns

# + colab={} colab_type="code" id="gvV9VORbxyvu"
df = pd.read_csv('https://archive.ics.uci.edu/ml/machine-learning-databases/blood-transfusion/transfusion.data')

df = df.rename(columns={
    'Recency (months)': 'months_since_last_donation', 
    'Frequency (times)': 'number_of_donations', 
    'Monetary (c.c. blood)': 'total_volume_donated', 
    'Time (months)': 'months_since_first_donation', 
    'whether he/she donated blood in March 2007': 'made_donation_in_march_2007'
})

# + colab={"base_uri": "https://localhost:8080/", "height": 202} colab_type="code" id="LRnIuYlm2o-K" outputId="4a6cc7ee-6718-4a79-ab9b-479b6079b96a"
df.head()

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" id="wETIyhc33JUL" outputId="8e4b7686-a468-4330-8818-61a6555ec668"
df.isnull().sum().sum()

# + colab={} colab_type="code" id="kysY3HHI_oc7"
X = df.drop(columns='made_donation_in_march_2007')
y = df.made_donation_in_march_2007

# + [markdown] colab_type="text" id="IxKfgx4ycb3c"
# ## Part 1.1 — Begin with baselines
#
# What **accuracy score** would you get here with a **"majority class baseline"?** 
#  
# (You don't need to split the data into train and test sets yet. You can answer this question either with a scikit-learn function or with a pandas function.)

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" id="VePhwhCi9-xR" outputId="1ef9c777-a1d0-47a8-d23c-e8b578cbd2bb"
# Finding Baseline Accuracy using Sklearn
import numpy as np
y_pred = np.full(shape=y.shape, fill_value=y.mode())# TODO
baseline_accuracy = accuracy_score(y, y_pred)
print ('Baseline Accuracy',baseline_accuracy)

# + colab={"base_uri": "https://localhost:8080/", "height": 104} colab_type="code" id="3oo31Remcq-x" outputId="19aeb551-8f92-4245-fc6f-f6dc11fa69c7"
# Finding Baseline Accuracy using Dummy Classifier

pipe = make_pipeline(
    DummyClassifier(strategy='most_frequent',random_state=42))

pipe.fit(X, y)

# Get the scores with the appropriate score function
# Predict with X features and Compare predictions to y labels
y_pred = pipe.predict(X)
dummy_score = accuracy_score(y, y_pred)
print(y.sum())
print(y_pred.sum())
print('Dummy Classification Score (Accuracy):', dummy_score)

# + [markdown] colab_type="text" id="_KdxE1TrcriI"
# What **recall score** would you get here with a **majority class baseline?**
#
# (You can answer this question either with a scikit-learn function or with no code, just your understanding of recall.)

# + colab={"base_uri": "https://localhost:8080/", "height": 138} colab_type="code" id="pLAy7cFh8EVf" outputId="4eb127b4-7c3d-4c22-bb3e-acb8a7042ef2"
# This question is tricky because using a baseline there would be no false negatives
print(f"Lets Check the value counts so we'll know what our majority class is:\n{y.value_counts()}\n")
print(f"Because our mode is zero and we don't have any false positives on the baseline our recall is zero true postives / (zero true positives + zero false positives))")

# Finding Baseline Recall using Sklearn
import numpy as np
y_pred = np.full(shape=y.shape, fill_value=y.mode())# TODO
baseline_recall = recall_score(y, y_pred)
print ('Baseline Recall',baseline_recall)

# + colab={"base_uri": "https://localhost:8080/", "height": 69} colab_type="code" id="ILS0fN0Cctyc" outputId="c26ee041-60af-4fba-a5d7-92decca640f9"
# Finding Baseline Recall Using Dummy Classifier


pipe = make_pipeline(
    DummyClassifier(strategy='most_frequent',random_state=42))

pipe.fit(X, y)

# Get the scores with the appropriate score function
# Predict with X features and Compare predictions to y labels
y_pred = pipe.predict(X)
dummy_score = recall_score(y, y_pred)
print('Dummy Classification Score (recall):', dummy_score)

# + [markdown] colab_type="text" id="QqYNDtwKYhji"
# ## Part 1.2 — Split data
#
# In this Sprint Challenge, you will use "Cross-Validation with Independent Test Set" for your model evaluation protocol.
#
# First, **split the data into `X_train, X_test, y_train, y_test`**, with random shuffle. (You can include 75% of the data in the train set, and hold out 25% for the test set.)
#

# + colab={"base_uri": "https://localhost:8080/", "height": 190} colab_type="code" id="mPKf86yDYf0t" outputId="7013045a-f6e9-4949-affd-bf0d7b672632"
from sklearn.model_selection import train_test_split
def split(X_values, y_values):
    # Hold out an "out-of-time" test set, from the last 100 days of data
    X_train, X_test, y_train, y_test = train_test_split(X_values, y_values, test_size=0.25, random_state=42)
    return X_train, X_test, y_train, y_test
  
X_train, X_test, y_train, y_test = split(X,y)
print ("Took this data...")
print (f'X Shape: {X.shape}\nY Shape: {y.shape}\n\n')
print ("And split it into this data... ")
print (f'X_train Shape: {X_train.shape},\nX_test Shape: {X_test.shape},\ny_train Shape: {y_train.shape},\ny_test Shape: {y_test.shape}')

# + [markdown] colab_type="text" id="E_ATNJdqTCuZ"
# ## Part 2.1 — Make a pipeline
#
# Make a **pipeline** which includes:
# - Preprocessing with any scikit-learn [**Scaler**](https://scikit-learn.org/stable/modules/classes.html#module-sklearn.preprocessing)
# - Feature selection with **[`SelectKBest`](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.SelectKBest.html)([`f_classif`](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.f_classif.html))**
# - Classification with [**`LogisticRegression`**](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html)

# + colab={"base_uri": "https://localhost:8080/", "height": 69} colab_type="code" id="8DRrVU5n5_Jw" outputId="e8af4ec5-d0ae-4da7-a9b9-dbeecf6d2f06"

pipe = make_pipeline(
    RobustScaler(), 
    SelectKBest(f_classif), 
    LogisticRegression())

print("Pipeline Created")

# + [markdown] colab_type="text" id="5vRkQHatglMG"
# ## Part 2.2 — Do Grid Search Cross-Validation
#
# Do [**GridSearchCV**](https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.GridSearchCV.html) with your pipeline. Use **5 folds** and **recall score**.
#
# Include these **parameters for your grid:**
#
# #### `SelectKBest`
# - `k : 1, 2, 3, 4`
#
# #### `LogisticRegression`
# - `class_weight : None, 'balanced'`
# - `C : .0001, .001, .01, .1, 1.0, 10.0, 100.00, 1000.0, 10000.0`
#
#
# **Fit** on the appropriate data.

# + colab={"base_uri": "https://localhost:8080/", "height": 138} colab_type="code" id="wgN8kG0ogBMH" outputId="5870e1a2-f3b2-4d67-dcc7-6a3577fbb3a4"

param_grid = {
    'selectkbest__k': [1,2,3,4], 
    'logisticregression__class_weight' : [None, 'balanced'],
    'logisticregression__C' : [0.0001, 0.001, 0.01, 0.1, 1.0, 10.0, 100.00, 1000.0, 10000.0]
}

# Fit on the train set, with grid search cross-validation
gs = GridSearchCV(pipe, param_grid=param_grid, cv=5, 
                  scoring='recall_weighted', 
                  verbose=1)
print("Grid Search Cross Validation now running...")
gs.fit(X_train, y_train)
print("Grid Search CV complete...")

# + [markdown] colab_type="text" id="urY_Wp3AiF83"
# ## Part 3 — Show best score and parameters
#
# Display your **best cross-validation score**, and the **best parameters** (the values of `k, class_weight, C`) from the grid search.
#
# (You're not evaluated here on how good your score is, or which parameters you find. You're only evaluated on being able to display the information. There are several ways you can get the information, and any way is acceptable.)

# + colab={"base_uri": "https://localhost:8080/", "height": 418} colab_type="code" id="qAxxkjG7gACP" outputId="8d58eee2-273f-4b67-c39b-406c62e10254"
# Scores for the Question's Answer
validation_score = gs.best_score_
print('***** Grid Search Scores *****')
print('\nBest Cross-Validation Score:', validation_score)
print('Best parameters:', gs.best_params_ ,'\n')

# Scores for me. 
print("\n***** A few more stats just for me *****")

# Predict with X_test features and compare to actual.
y_pred_train = gs.predict(X_train)
train_score_A = accuracy_score(y_train, y_pred_train)
print('Train Score (Accuracy):', train_score_A)
y_pred_test = gs.predict(X_test)
test_score_A = accuracy_score(y_test, y_pred_test)
print('Test Score (Accuracy):', test_score_A)

train_score_B = gs.score(X_train, y_train)
print('\nTrain Score ("Recall"): ', train_score_B)
test_score_B = gs.score(X_test, y_test)
print('Test Score ("Recall"): ', test_score_B)
print('\nBest estimator:\n', gs.best_estimator_)
cvresults = pd.DataFrame(gs.cv_results_)
print('\nGenerated Results with Shape:', cvresults.shape)

# + [markdown] colab_type="text" id="jkyHoRIbEgRR"
# ## Part 4 — Calculate classification metrics from a confusion matrix
#
# Suppose this is the confusion matrix for your binary classification model:
#
# <table>
#   <tr>
#     <th colspan="2" rowspan="2"></th>
#     <th colspan="2">Predicted</th>
#   </tr>
#   <tr>
#     <th>Negative</th>
#     <th>Positive</th>
#   </tr>
#   <tr>
#     <th rowspan="2">Actual</th>
#     <th>Negative</th>
#     <td>85</td>
#     <td>58</td>
#   </tr>
#   <tr>
#     <th>Positive</th>
#     <td>8</td>
#     <td>36</td>
#   </tr>
# </table>

# + colab={} colab_type="code" id="Npu-IpcUMf3x"
true_negative = 85
true_positive = 36
false_negative = 8
false_positive = 58

# + [markdown] colab_type="text" id="LhyMM5H-JpVB"
# Calculate accuracy

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" id="TZPwqdh2KUcB" outputId="71452d23-132a-4f77-bdeb-9f0c59042575"
accuracy = (true_negative + true_positive) / (true_negative + true_positive + false_negative + false_positive)
print(accuracy)

# + [markdown] colab_type="text" id="BRWLfGcGKeQw"
# Calculate precision

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" id="A-FEZ4i_Kf_n" outputId="b51f829e-08e6-442d-a3d9-3441ce7275d7"
precision = true_positive / (true_positive + false_positive)
print(precision)

# + [markdown] colab_type="text" id="h_mH2NYDKi2C"
# Calculate recall

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" id="U4_wJGyjKkXJ" outputId="5156b333-1d53-40dc-c06b-f90c33ec5e3d"
recall = true_positive / (true_positive + false_negative)
print(recall)

# + [markdown] colab_type="text" id="9KEaWsk5Kk9W"
# ## BONUS — How you can earn a score of 3
#
#

# + [markdown] colab_type="text" id="Nd1EeA8uT5EZ"
# ### Part 1
# Do feature engineering, to try improving your cross-validation score.
#
#

# + colab={} colab_type="code" id="QsVrYUwQUBf0"
df["new_variable"] = (df["months_since_first_donation"] - df["months_since_last_donation"])

# + colab={"base_uri": "https://localhost:8080/", "height": 554} colab_type="code" id="OXsWEFLeUanC" outputId="4b3d293b-75ad-4fe4-8319-a5c8ba17e668"
X = df.drop(columns='made_donation_in_march_2007')
y = df.made_donation_in_march_2007

# Split Data
X_train, X_test, y_train, y_test = split(X,y)
print ("Took this data...")
print (f'X Shape: {X.shape}\nY Shape: {y.shape}\n\n')
print ("And split it into this data... ")
print (f'X_train Shape: {X_train.shape},\nX_test Shape: {X_test.shape},\ny_train Shape: {y_train.shape},\ny_test Shape: {y_test.shape}')

# Make Pipeline
pipe = make_pipeline(
    RobustScaler(), 
    SelectKBest(f_classif), 
    LogisticRegression())

print("\nPipeline Created")

# GridSearch CV
param_grid = {
    'selectkbest__k': [1,2,3,4], 
    'logisticregression__class_weight' : [None, 'balanced'],
    'logisticregression__C' : [0.0001, 0.001, 0.01, 0.1, 1.0, 10.0, 100.00, 1000.0, 10000.0]
}

# Fit on the train set, with grid search cross-validation
gs = GridSearchCV(pipe, param_grid=param_grid, cv=5, 
                  scoring='recall_weighted', 
                  verbose=1)
print("\nGrid Search Cross Validation now running...")
gs.fit(X_train, y_train)
print("Grid Search CV complete...")

# Check out the scores
new_validation_score = gs.best_score_
print('\n***** Grid Search Scores *****')
print('New Cross-Validation Score:', new_validation_score)
print('Previous Cross-Validation Score', validation_score)
print('Best parameters:', gs.best_params_ ,'\n')

# Scores for me. 
print("\n***** A few more stats just for me *****")

# Predict with X_test features and compare to actual.
y_pred_train = gs.predict(X_train)
train_score_C = accuracy_score(y_train, y_pred_train)
print('Train Score (Accuracy):', train_score_C)
y_pred_test = gs.predict(X_test)
test_score_C = accuracy_score(y_test, y_pred_test)
print('Test Score (Accuracy):', test_score_C)

train_score_D = gs.score(X_train, y_train)
print('\nTrain Score ("Recall"): ', train_score_D)
test_score_D = gs.score(X_test, y_test)
print('Test Score ("Recall"): ', test_score_D)

# + [markdown] colab_type="text" id="vQCjItwWT6ks"
# ### Part 2
# Add transformations in your pipeline and parameters in your grid, to try improving your cross-validation score.
#

# + colab={"base_uri": "https://localhost:8080/", "height": 121} colab_type="code" id="hAXH5DU7UA4s" outputId="2e02afc7-05a1-4249-932a-1af3044a410b"
# I'm going to try a slightly different one this time. I want to try random forest classifier. 

# Make Pipeline
pipe = make_pipeline(
    RobustScaler(), 
    SelectKBest(f_classif), 
    RandomForestClassifier())

print("\nPipeline Created")

# GridSearch CV
param_grid = {
    'selectkbest__k': [1,2,3,4], 
    "randomforestclassifier__max_depth": [80, 90, 100, 110],
#    "randomforestclassifier__max_features": [2, 3],
    "randomforestclassifier__min_samples_split": [8, 10, 12],
    "randomforestclassifier__min_samples_leaf": [3, 4, 5],
    "randomforestclassifier__bootstrap": [False],
    "randomforestclassifier__n_estimators" :[100, 200, 300, 1000],
    "randomforestclassifier__criterion": ["gini"]}


# Fit on the train set, with grid search cross-validation
gs = GridSearchCV(pipe, param_grid=param_grid, cv=10, 
                  scoring='recall_weighted', 
                  verbose=1, n_jobs=10)

print("\nGrid Search Cross Validation now running...")
gs.fit(X_train, y_train)
print("Grid Search CV complete...")

# Check out the scores
new_validation_score = gs.best_score_
print('\n***** Grid Search Scores *****')
print('New Cross-Validation Score:', new_validation_score)
print('Previous Cross-Validation Score', validation_score)
print('Best parameters:', gs.best_params_ ,'\n')
print('Best score:', gs.best_score_)
print('Best estimator:', gs.best_estimator_)


# -

# #### Lets see how it's test stats turned out

# +
y_pred_train = gs.predict(X_train)
train_score_C = accuracy_score(y_train, y_pred_train)
print('Random Forest Classifier - Train Score (Accuracy):', train_score_C)
y_pred_test = gs.predict(X_test)
test_score_C = accuracy_score(y_test, y_pred_test)
print('Random Forest Classifier - Test Score (Accuracy):', test_score_C)

train_score_D = gs.score(X_train, y_train)
print('\nRandom Forest Classifier - Train Score ("Recall"): ', train_score_D)
test_score_D = gs.score(X_test, y_test)
print('Random Forest Classifier - Test Score ("Recall"): ', test_score_D)

# + [markdown] colab_type="text" id="uHQGZ8pJT9Gl"
# ### Part 3
# Show names of selected features. Then do a final evaluation on the test set — what is the test score?
#

# + colab={} colab_type="code" id="ErfSIVPGT-5U"
# Which features were selected?
# 'selectkbest' is the autogenerated name of the SelectKBest() function in the pipeline
selector = gs.best_estimator_.named_steps['selectkbest']
all_names = X_train.columns

# get_support returns a mask of the columns in True / False
selected_mask = selector.get_support()
# Passing the boolean list as the column names creates a 
selected_names = all_names[selected_mask]
unselected_names = all_names[~selected_mask]

print('Features selected:')
for name in selected_names:
    print(name)

print()
print('Features not selected:')
for name in unselected_names:
    print(name)

# + [markdown] colab_type="text" id="P2bN-OzZT_We"
#
# ### Part 4
# Calculate F1 score and False Positive Rate. 

# + colab={} colab_type="code" id="2bIQrqcuT_wU"
from sklearn.metrics import confusion_matrix
y_pred_train = gs.predict(X_train)
train_score_F1 = f1_score(y_train, y_pred_train)
print('Random Forest Classifier - F1 Train Score:', train_score_F1)
y_pred_test = gs.predict(X_test)
test_score_F1 = f1_score(y_test, y_pred_test)
print('Random Forest Classifier - F1 Test Score:', test_score_F1)

tn, fp, fn, tp = confusion_matrix(y_train, y_pred_train).ravel()
print(f'Random Forest Classifier - Train False Positive Rate: {fp / (fp + tp)}')

tn, fp, fn, tp = confusion_matrix(y_test, y_pred_test).ravel()
print(f'Random Forest Classifier - Test False Positive Rate: {fp / (fp + tp)}')

StarcoderdataJupyterScriptsDedupFiltered