<jupyter_start><jupyter_text>1. Logistic Regression<jupyter_code>lm = LogisticRegression()
lm.fit(X_train, y_train)
X_test = scaler.transform(df_test[features])
y_test = df_test['target']
preds = lm.predict(X_test)
# Threshold = 0.5
pd.DataFrame(confusion_matrix(df_test['target'], preds),\
             columns=['Predict-not changing (0)','Predict-change (1)'],\
            index=['Not changing (0)','Changing (1)'])
roc_auc_score(y_test,preds)
# Cross-validation
from sklearn.model_selection import cross_val_score
cv_scores = cross_val_score(lm,X_test, y_test, cv=5, scoring='roc_auc')
print('Scores:', cv_scores)
print('Mean:', np.mean(cv_scores))<jupyter_output>Scores: [0.74089266 0.7017615  0.69201734 0.69766532 0.69147265]
Mean: 0.7047618917177652<jupyter_text>Explore threshold values<jupyter_code>def make_confusion_matrix(model, threshold=0.5):
    # Predict class 1 if probability of being in class 1 is greater than threshold
    # (model.predict(X_test) does this automatically with a threshold of 0.5)
    y_predict = (model.predict_proba(X_test)[:, 1] >= threshold)
    fraud_confusion = confusion_matrix(y_test, y_predict)
    plt.figure(dpi=80)
    sns.heatmap(fraud_confusion, cmap=plt.cm.Blues, annot=True, square=True, fmt='d',
           xticklabels=['Not Looking', 'Looking'],
           yticklabels=['Not Looking', 'Looking']);
    plt.xlabel('prediction')
    plt.ylabel('actual')
# Let's see how our confusion matrix changes with changes to the cutoff! 

from ipywidgets import interactive, FloatSlider
interactive(lambda threshold: make_confusion_matrix(lm, threshold), threshold=(0.0,1.0,0.02))
df_test['proba_change'] = lm.predict_proba(X_test)[:, 1]

fpr, tpr, thresholds = roc_curve(df_test['target'],
                                 df_test['proba_change'])
def plot_roc(true, probas):
    auc = roc_auc_score(true, probas)

    plt.plot(fpr, tpr, marker='o')
    plt.xlabel('1 - Specificity (FPR)')
    plt.ylabel('Sensitivity (TPR)');
    plt.title(f"Area Under the ROC Curve: {round(auc, 3)}");
plot_roc(df_test['target'], lm.predict_proba(X_test)[:, 1])
mask = tpr > 0.95
thresholds[mask].max()
plot_roc(df_test['target'], lm.predict_proba(X_test)[:, 1] >= 0.113)
preds_low_th = (lm.predict_proba(X_test)[:, 1] >= 0.2)

# Threshold = 0.146
pd.DataFrame(confusion_matrix(df_test['target'], preds_low_th),\
             columns=['Predict-not changing (0)','Predict-change (1)'],\
            index=['Not changing (0)','Changing (1)'])
print(classification_report(df_test['target'],np.array(preds_low_th)))
# Precision, recall curve
precision, recall, thresholds = precision_recall_curve(y_test, lm.predict_proba(X_test)[:,1])
plt.plot(thresholds, precision[1:],label='Precision')
plt.plot(thresholds, recall[1:],label='Recall');

# Plot random guess precision, recall currve
no_skill = len(y_test[y_test==1]) / len(y_test)
plt.plot([0,1], [no_skill,no_skill], linestyle='--', label='Random')

plt.legend(loc='lower left')
plt.xlabel('Threshold (above this probability, label as Looking for change)');
plt.title('Precision and Recall Curves');
plt.plot(recall, precision, marker='.', label='Logistic')

# Plot random guess precision, recall currve
no_skill = len(y_test[y_test==1]) / len(y_test)
plt.plot([0,1], [no_skill,no_skill], linestyle='--', label='Random')

plt.xlabel('Recall')
plt.ylabel('Precision')
plt.legend();
# ROC AUC score
score_default = roc_auc_score(y_test, lm.predict(X_test)) # Threshold = 0.5
score_low_th = roc_auc_score(y_test, lm.predict_proba(X_test)[:, 1] > 0.113) # Threshold = 0.113
print('ROC AUC score (threshold = 0.5):', score_default)
print('ROC AUC score (threshold = 0.113):', score_low_th)
# F1 score
score_default = f1_score(y_test, lm.predict(X_test)) # Threshold = 0.5
score_low_th = f1_score(y_test, lm.predict_proba(X_test)[:, 1] > 0.113) # Threshold = 0.113
print('F1 score (threshold = 0.5):', score_default)
print('F1 score (threshold = 0.113):', score_low_th)
# Log loss score
score_default = log_loss(y_test, lm.predict(X_test)) # Threshold = 0.5
score_low_th = log_loss(y_test, lm.predict_proba(X_test)[:, 1] > 0.113) # Threshold = 0.113
print('Log loss (threshold = 0.5):', score_default)
print('Log loss (threshold = 0.113):', score_low_th)<jupyter_output>Log loss (threshold = 0.5): 8.014204741948493
Log loss (threshold = 0.113): 23.13452934539692<jupyter_text>2. KNN<jupyter_code>from sklearn import preprocessing

le = preprocessing.LabelEncoder()
from sklearn.neighbors import KNeighborsClassifier

knn_3 = KNeighborsClassifier(n_neighbors=3)
knn_3.fit(X_train,y_train)
pred = knn_3.predict(X_test)

print('Accuracy score:',accuracy_score(y_test,pred),'\n')
print('Classification report:\n\n',classification_report(y_test,pred))
knn_5 = KNeighborsClassifier(n_neighbors=5)
knn_5.fit(X_train,y_train)
pred = knn_5.predict(X_test)

print('Accuracy score:',accuracy_score(y_test,pred),'\n')
print('Classification report:\n\n',classification_report(y_test,pred))
knn_7 = KNeighborsClassifier(n_neighbors=7)
knn_7.fit(X_train,y_train)
pred = knn_7.predict(X_test)

print('Accuracy score:',accuracy_score(y_test,pred),'\n')
print('Classification report (kNN):\n\n',classification_report(y_test,pred))
# knn (n=3)
y_scores = knn_3.predict_proba(X_test)
fpr, tpr, threshold = roc_curve(y_test, y_scores[:, 1])
roc_auc = auc(fpr, tpr)
plt.plot(fpr, tpr, 'b', label = 'AUC_knn3 = %0.2f' % roc_auc)

# knn (n=5)
y_scores = knn_5.predict_proba(X_test)
fpr, tpr, threshold = roc_curve(y_test, y_scores[:, 1])
roc_auc = auc(fpr, tpr)
plt.plot(fpr, tpr, 'g', label = 'AUC_knn5 = %0.2f' % roc_auc)

# knn (n=7)
y_scores = knn_7.predict_proba(X_test)
fpr, tpr, threshold = roc_curve(y_test, y_scores[:, 1])
roc_auc = auc(fpr, tpr)
plt.plot(fpr, tpr, 'k', label = 'AUC_knn7 = %0.2f' % roc_auc)


plt.legend(loc = 'lower right')
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.title('Receiver Operating Characteristic')
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.title('ROC Curve of kNN')
plt.show()<jupyter_output><empty_output><jupyter_text>3. Naive Bayes<jupyter_code>from sklearn.naive_bayes import GaussianNB

nb = GaussianNB()
nb.fit(X_train,y_train)
pred = nb.predict(X_test)
print('Accuracy score:',accuracy_score(y_test,pred),'\n')
print('Classification report (Naive Bayes):\n\n',classification_report(y_test,pred))
# Naive Bayes
y_scores = nb.predict_proba(X_test)
fpr, tpr, threshold = roc_curve(y_test, y_scores[:, 1])
roc_auc = auc(fpr, tpr)
plt.plot(fpr, tpr, 'b', label = 'AUC_nb = %0.2f' % roc_auc)

plt.legend(loc = 'lower right')
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.title('Receiver Operating Characteristic')
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.title('ROC Curve of Naive Bayes')
plt.show()<jupyter_output><empty_output>