<jupyter_start><jupyter_text>train test split data<jupyter_code>winData[train].head()
X_train, X_test, y_train, y_test = train_test_split(winData[train],
                                                    np.ravel(winData[target]), random_state=42, 
                                                    test_size=0.3, stratify=winData.win)
# design model
model_lr = LogisticRegression(solver="lbfgs")
model_cat = CatBoostClassifier(iterations=3000, random_state=42, 
                              logging_level="Silent")

# fit model
model_lr.fit(X_train,y_train)
model_cat.fit(X_train,y_train, plot=True)

# predict with model
y_pred_lr = model_lr.predict(X_test)
y_pred_cat = model_cat.predict(X_test)

# predict with probabilites
y_pred_proba_lr = model_lr.predict_proba(X_test)
y_pred_proba_cat = model_cat.predict_proba(X_test)
plt.plot(y_pred_proba_lr[:,1])
plt.plot(y_pred_proba_cat[:,1])<jupyter_output><empty_output><jupyter_text>Eval model<jupyter_code>from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import roc_curve
from sklearn.metrics import auc<jupyter_output><empty_output><jupyter_text>Accuracy<jupyter_code>print("lr: ", accuracy_score(y_pred_lr, y_test))
print("cat: ", accuracy_score(y_pred_cat, y_test))

#print("pred LR mean: ", np.mean(y_pred_lr))<jupyter_output>lr:  0.9813486370157819
cat:  0.9813486370157819<jupyter_text>Confusion matrix<jupyter_code>confusion_matrix(y_pred_lr,y_test)<jupyter_output><empty_output><jupyter_text>ROC curve<jupyter_code>def rocPlot(y_test, y_pred_list):
    plt.style.use('fivethirtyeight')
    plt.figure(figsize=[8,8])
    lw=5
    
    for y_pred in y_pred_list:
        fpr, tpr, _ = roc_curve(y_test, y_pred[:,1])
        roc_auc = auc(fpr, tpr)
        
        plt.plot(fpr, tpr,
        lw=lw, label="ROC curve (area = %0.2f)" % roc_auc)
    
    plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('Receiver operating characteristic')
    plt.legend(loc="lower right")
    pass

    return(0)
rocPlot(y_test,[y_pred_proba_lr, y_pred_proba_cat])<jupyter_output><empty_output><jupyter_text>Find the optimal threshold<jupyter_code>def Find_Optimal_Cutoff(target, predicted):
    """ Find the optimal probability cutoff point for a classification model related to event rate
    Parameters
    ----------
    target : Matrix with dependent or target data, where rows are observations

    predicted : Matrix with predicted data, where rows are observations

    Returns
    -------     
    list type, with optimal cutoff value

    """
    fpr, tpr, threshold = roc_curve(target, predicted)
    optimal_idx = np.argmin(np.abs(tpr - fpr)) # Edit: Change to argmin!
    optimal_threshold = threshold[optimal_idx]
    
    i = np.arange(len(tpr)) 
    roc = pd.DataFrame({'tf' : pd.Series(tpr-(1-fpr), index=i), 'threshold' : pd.Series(threshold, index=i)})
    roc_t = roc.iloc[np.abs(roc.tf-0).argsort()[:1]]

    return list(roc_t['threshold']) 
threshold_lr = Find_Optimal_Cutoff(y_test, y_pred_proba_lr[:,1])
print("lr: ", threshold_lr)
threshold_cat = Find_Optimal_Cutoff(y_test, y_pred_proba_cat[:,1])
print("cat:",threshold_cat)
data = pd.DataFrame({"target": y_test, 
                     "pred_proba_lr": y_pred_proba_lr[:,1],
                     "pred_proba_cat": y_pred_proba_lr[:,1]})

# Find prediction to the dataframe applying threshold
data['pred_lr'] = data['pred_proba_lr'].map(lambda x: 1 if x > threshold_lr[0] else 0)
data['pred_cat'] = data['pred_proba_cat'].map(lambda x: 1 if x > threshold_cat[0] else 0)

# Print confusion Matrix
print("lr: \n",confusion_matrix(data['target'], data['pred_lr']))
print("\ncat: \n",confusion_matrix(data['target'], data['pred_cat']))

plt.plot(data["pred_lr"],'r*', alpha=0.5)
plt.plot(data["pred_cat"],'b^', alpha=0.5)
plt.plot(data["target"],'g.')<jupyter_output><empty_output><jupyter_text>How do we perform on year 2018?<jupyter_code># get from 2018
win2018 = winData_all.loc[winData_all.year==2018,:]

X_2018 = win2018[train]
y_2018 = win2018[target]

pred_2018 = model_lr.predict_proba(X_2018)[:,1]
X_2018["country_id"] = win2018.country_id
X_2018["country_name"] = countries.iloc[win2018.country_id].get_values()
X_2018["target"] = y_2018
X_2018["pred"] = pred_2018
X_2018.head()
pred10 = X_2018.sort_values("pred", ascending=False)["country_name"].head(10).get_values()
pred10
placement_df = pd.read_csv("../DATA/placement_df.csv")
result2018 = placement_df.iloc[-1,]
res10 = result2018.sort_values().head(10).index.get_values()
res10<jupyter_output><empty_output><jupyter_text>What is the overlap between predicted top 10 and actual top 10?<jupyter_code>np.isin(pred10,res10).mean()<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup