# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

# #Exploring Ensemble Methods

# In this homework we will explore the use of boosting. You will:
#
# Use SFrames to do some feature engineering.
# Train a boosted ensemble of decision-trees (gradient boosted trees) on the lending club dataset.
# Predict whether a loan will default along with prediction probabilities (on a validation set).
# Evaluate the trained model and compare it with a baseline.
# Find the most positive and negative loans using the learned model.
# Explore how the number of trees influences classification performance.
#

# #Load the Lending Club dataset

import pandas as pd
import numpy as np

loans = pd.read_csv('/Users/April/Downloads/lending-club-data.csv')

# safe_loans =  1 => safe
# safe_loans = -1 => risky
loans['safe_loans'] = loans['bad_loans'].apply(lambda x : +1 if x==0 else -1)
loans = loans.drop('bad_loans', axis = 1)

# Selecting features
# The features we will be using are described in the code comments below. Extract these feature columns and target column from the dataset. We will only use these features.

target = 'safe_loans'
features = ['grade',                     # grade of the loan (categorical)
            'sub_grade_num',             # sub-grade of the loan as a number from 0 to 1
            'short_emp',                 # one year or less of employment
            'emp_length_num',            # number of years of employment
            'home_ownership',            # home_ownership status: own, mortgage or rent
            'dti',                       # debt to income ratio
            'purpose',                   # the purpose of the loan
            'payment_inc_ratio',         # ratio of the monthly payment to income
            'delinq_2yrs',               # number of delinquincies
             'delinq_2yrs_zero',          # no delinquincies in last 2 years
            'inq_last_6mths',            # number of creditor inquiries in last 6 months
            'last_delinq_none',          # has borrower had a delinquincy
            'last_major_derog_none',     # has borrower had 90 day or worse rating
            'open_acc',                  # number of open credit accounts
            'pub_rec',                   # number of derogatory public records
            'pub_rec_zero',              # no derogatory public records
            'revol_util',                # percent of available credit being used
            'total_rec_late_fee',        # total late fees received to day
            'int_rate',                  # interest rate of the loan
            'total_rec_int',             # interest received to date
            'annual_inc',                # annual income of borrower
            'funded_amnt',               # amount committed to the loan
            'funded_amnt_inv',           # amount committed by investors for the loan
            'installment',               # monthly payment owed by the borrower
           ]

# #Skipping observations with missing values

# Recall from the lectures that one common approach to coping with missing values is to skip observations that contain missing values.

loans = loans[[target] + features].dropna()

loans = pd.get_dummies(loans)

import json
with open('/Users/April/Desktop/datasci_course_materials-master/assignment1/train index.json', 'r') as f: # Reads the list of most frequent words
    train_idx = json.load(f)
with open('/Users/April/Desktop/datasci_course_materials-master/assignment1/validation index.json', 'r') as f1: # Reads the list of most frequent words
    validation_idx = json.load(f1)

train_data = loans.iloc[train_idx]
validation_data = loans.iloc[validation_idx]

# #Gradient boosted tree classifier

# Now, let's use the built-in scikit learn gradient boosting classifier (sklearn.ensemble.GradientBoostingClassifier) to create a gradient boosted classifier on the training data. You will need to import sklearn, sklearn.ensemble, and numpy.

# You will have to first convert the SFrame into a numpy data matrix. See the API for more information. You will also have to extract the label column. Make sure to set max_depth=6 and n_estimators=5.

# #Making predictions

# Just like we did in previous sections, let us consider a few positive and negative examples from the validation set. We will do the following:
#
# Predict whether or not a loan is likely to default.
# Predict the probability with which the loan is likely to default.

# First, let's grab 2 positive examples and 2 negative examples.

# +
validation_safe_loans = validation_data[validation_data[target] == 1]
validation_risky_loans = validation_data[validation_data[target] == -1]

sample_validation_data_risky = validation_risky_loans[0:2]
sample_validation_data_safe = validation_safe_loans[0:2]

sample_validation_data = sample_validation_data_safe.append(sample_validation_data_risky)
sample_validation_data
# -

# For each row in the sample_validation_data, write code to make model_5 predict whether or not the loan is classified as a safe loan. (Hint: if you are using scikit-learn, you can use the .predict() method)

import sklearn
import sklearn.ensemble
import numpy

from sklearn.ensemble import GradientBoostingClassifier

sample_model = GradientBoostingClassifier(n_estimators=5, max_depth=6)

X = train_data.drop('safe_loans',1)

X.columns

sample_model.fit(X, train_data['safe_loans'])

sample_model.predict(sample_validation_data.drop('safe_loans',1))

# Quiz question: What percentage of the predictions on sample_validation_data did model_5 get correct?

# #Prediction Probabilities

# For each row in the sample_validation_data, what is the probability (according model_5) of a loan being classified as safe? (Hint: if you are using scikit-learn, you can use the .predict_proba() method)

sample_model.predict_proba(sample_validation_data.drop('safe_loans',1))

# Quiz Question: Which loan has the highest probability of being classified as a safe loan?
#
# Checkpoint: Can you verify that for all the predictions with probability >= 0.5, the model predicted the label +1?

# #Evaluating the model on the validation data

# Evaluate the accuracy of the model_5 on the validation_data. (Hint: if you are using scikit-learn, you can use the .score() method)

sample_model.score(validation_data.drop('safe_loans',1), validation_data['safe_loans'])

# Calculate the number of false positives made by the model on the validation_data.

predict_safeloans = sample_model.predict(validation_data.drop('safe_loans',1))

predict_safeloans

sum(predict_safeloans > validation_data['safe_loans'])

# #Comparison with decision trees

# false negative
sum(predict_safeloans < validation_data['safe_loans'])

# Quiz Question: Using the same costs of the false positives and false negatives, what is the cost of the mistakes made by the boosted tree model (model_5) as evaluated on the validation_set?

cost = 20000*1653+10000*1491
print cost

# #Most positive & negative loans

# In this section, we will find the loans that are most likely to be predicted safe.

validation_data['predictions'] = sample_model.predict_proba(validation_data.drop('safe_loans',1))[:,1]

validation_data[['grade_A','grade_B','grade_C','grade_D','predictions']].sort('predictions', ascending = False).head(5)

validation_data[['grade_A','grade_B','grade_C','grade_D','predictions']].sort('predictions', ascending = False).tail(5)

# #Effects of adding more trees

# In this assignment, we will train 5 different ensemble classifiers in the form of gradient boosted trees.

# Train models with 10, 50, 100, 200, and 500 trees. Use the n_estimators parameter to control the number of trees. Remember to keep max_depth = 6.
#
# Call these models model_10, model_50, model_100, model_200, and model_500, respectively. This may take a few minutes to run.

model_10 = GradientBoostingClassifier(n_estimators=10, max_depth=6)
model_10.fit(train_data.drop('safe_loans',1), train_data['safe_loans'])

model_50 = GradientBoostingClassifier(n_estimators=50, max_depth=6)
model_50.fit(train_data.drop('safe_loans',1), train_data['safe_loans'])

model_100 = GradientBoostingClassifier(n_estimators=100, max_depth=6)
model_100.fit(train_data.drop('safe_loans',1), train_data['safe_loans'])

model_200 = GradientBoostingClassifier(n_estimators=200, max_depth=6)
model_200.fit(train_data.drop('safe_loans',1), train_data['safe_loans'])

model_500 = GradientBoostingClassifier(n_estimators=500, max_depth=6)
model_500.fit(train_data.drop('safe_loans',1), train_data['safe_loans'])

model_10.score(validation_data.drop(['safe_loans','predictions'],1), validation_data['safe_loans'])

model_50.score(validation_data.drop(['safe_loans','predictions'],1), validation_data['safe_loans'])

model_100.score(validation_data.drop(['safe_loans','predictions'],1), validation_data['safe_loans'])

model_200.score(validation_data.drop(['safe_loans','predictions'],1), validation_data['safe_loans'])

model_500.score(validation_data.drop(['safe_loans','predictions'],1), validation_data['safe_loans'])

# #Plot the training and validation error vs. number of trees

# In this section, we will plot the training and validation errors versus the number of trees to get a sense of how these models are performing. We will compare the 10, 50, 100, 200, and 500 tree models. You will need matplotlib in order to visualize the plots.

import matplotlib.pyplot as plt
# %matplotlib inline
def make_figure(dim, title, xlabel, ylabel, legend):
    plt.rcParams['figure.figsize'] = dim
    plt.title(title)
    plt.xlabel(xlabel)
    plt.ylabel(ylabel)
    if legend is not None:
        plt.legend(loc=legend, prop={'size':15})
    plt.rcParams.update({'font.size': 16})
    plt.tight_layout()


# Steps to follow:
#
# Step 1: Calculate the classification error for each model on the training data (train_data).
# Step 2: Store the training errors into a list (called training_errors) that looks like this: [train_err_10, train_err_50, ..., train_err_500]
# Step 3: Calculate the classification error of each model on the validation data (validation_data).
# Step 4: Store the validation classification error into a list (called validation_errors) that looks like this:[validation_err_10, validation_err_50, ..., validation_err_500]
#

train_err_10 = 1 - model_10.score(train_data.drop('safe_loans',1), train_data['safe_loans'])

train_err_50 = 1 - model_50.score(train_data.drop('safe_loans',1), train_data['safe_loans'])

train_err_100 = 1 - model_100.score(train_data.drop('safe_loans',1), train_data['safe_loans'])

train_err_200 = 1 - model_200.score(train_data.drop('safe_loans',1), train_data['safe_loans'])

train_err_500 = 1 - model_500.score(train_data.drop('safe_loans',1), train_data['safe_loans'])

validation_err_10 = 1 - model_10.score(validation_data.drop(['safe_loans','predictions'],1), validation_data['safe_loans'])
validation_err_50 = 1 - model_50.score(validation_data.drop(['safe_loans','predictions'],1), validation_data['safe_loans'])
validation_err_100 = 1 - model_100.score(validation_data.drop(['safe_loans','predictions'],1), validation_data['safe_loans'])
validation_err_200 = 1 - model_200.score(validation_data.drop(['safe_loans','predictions'],1), validation_data['safe_loans'])
validation_err_500 = 1 - model_500.score(validation_data.drop(['safe_loans','predictions'],1), validation_data['safe_loans'])

training_errors = [train_err_10, train_err_50, train_err_100, train_err_200, train_err_500]

validation_errors = [validation_err_10, validation_err_50, validation_err_100, validation_err_200, validation_err_500]

# +
plt.plot([10, 50, 100, 200, 500], training_errors, linewidth=4.0, label='Training error')
plt.plot([10, 50, 100, 200, 500], validation_errors, linewidth=4.0, label='Validation error')

make_figure(dim=(10,5), title='Error vs number of trees',
            xlabel='Number of trees',
            ylabel='Classification error',
            legend='best')
# -



StarcoderdataJupyterScriptsDedupFiltered