# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: imbalanced
#     language: python
#     name: imbalanced
# ---

# ## Probability Calibration Curve
#
# In this notebook, we will plot Probability calibration curves for logistic regression predictions made from balanced and imbalanced datasets.
#
# Probability calibration curves allow us to visualize if the probability is calibrated.
#
# Remember that logistic regression tends to return balanced probabilities.

# +
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

# to create artificial data
from sklearn.datasets import make_classification

# to train a model and plot the calibration curve
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.calibration import calibration_curve

import warnings
warnings.filterwarnings("ignore")


# -

# ## Function to create toy data

def create_dataset(weights):

    # returns arrays
    X, y = make_classification(
        n_samples=10000,
        n_features=10,
        n_redundant=0,
        n_clusters_per_class=1,
        weights=[weights],  # to balance (or not) the classes
        class_sep=1,  # how separated the classes are
        random_state=1)

    # trasform arrays into pandas df and series
    X = pd.DataFrame(X)
    y = pd.Series(y)
    
    return X, y


# ## Create a balanced dataset
#
# The 2 classes are fairly well separated for the demonstration.

# +
X, y = create_dataset(weights=0.5)

# show dataset
X.head()

# +
# number of observations per class

y.value_counts()

# +
# balancing ratio

y.value_counts() / len(y)
# -

# ### Separate into train and test sets

# +
# separate dataset into train and test

X_train, X_test, y_train, y_test = train_test_split(
    X,  
    y,  
    test_size=0.3,
    random_state=0)

X_train.shape, X_test.shape

# +
# balancing ratio: train set

y_train.value_counts() / len(X_train)

# +
# number of observations per class

y_train.value_counts()
# -

# You can go ahead and check the balancing ratio for the test set, should be similar.
#
# ### Train Logistic Regression

# +
# Logistic Regression on balanced dataset

logit = LogisticRegression(
    penalty='l2',
    solver='newton-cg',
    C=1,
    random_state=0,
    max_iter=100,
).fit(X_train, y_train)

# score the test set
probs = logit.predict_proba(X_test)[:, 1]
# -

# ## Calibration Curve
#
# https://scikit-learn.org/stable/modules/generated/sklearn.calibration.calibration_curve.html#sklearn.calibration.calibration_curve
#
# - Sorts the probability values into bins
# - Determines the true positive fraction in those bins

# +
fraction_of_positives, mean_predicted_value = calibration_curve(
    y_test, probs, n_bins=10, strategy='uniform')

len(mean_predicted_value), len(fraction_of_positives)

# +
# mean probability of each interval / bin

mean_predicted_value

# +
# fraction of observations of the minority class
# in each interval of probability

fraction_of_positives

# +
# plot calibration curve
plt.plot(mean_predicted_value, fraction_of_positives, label='Logistic Regression')

# plot perfect calibration line
plt.plot(np.linspace(0, 1, 10), np.linspace(0, 1, 10),
         linestyle='--', color='red', label='Perfect calibration')

# add labels and legends
plt.xlabel('Probability Predictions')
plt.ylabel('Fraction of positive examples')
plt.legend(loc='upper left')
plt.show()

# +
# plot number of observation per prediction interval

plt.hist(probs, range=(0, 1), bins=10, density=True, stacked=True, alpha=0.3)
plt.show()


# +
# bring all together

def plot_calibration_curve(y_true, probs, bins, strategy):

    fraction_of_positives, mean_predicted_value = calibration_curve(
        y_true, probs, n_bins=bins, strategy=strategy)
    
    max_val = max(mean_predicted_value)
    
    plt.figure(figsize=(8,10))
    plt.subplot(2, 1, 1)
    plt.plot(mean_predicted_value, fraction_of_positives, label='Logistic Regression')
    plt.plot(np.linspace(0, max_val, bins), np.linspace(0, max_val, bins),
         linestyle='--', color='red', label='Perfect calibration')
    
    plt.xlabel('Probability Predictions')
    plt.ylabel('Fraction of positive examples')
    plt.title('Calibration Curve')
    plt.legend(loc='upper left')


    plt.subplot(2, 1, 2)
    plt.hist(probs, range=(0, 1), bins=bins, density=True, stacked=True, alpha=0.3)
    plt.xlabel('Probability Predictions')
    plt.ylabel('Fraction of examples')
    plt.title('Density')
    plt.show()
# -

plot_calibration_curve(y_test, probs, bins=10, strategy='uniform')

# ### Change number of probability bins

plot_calibration_curve(y_test, probs, bins=50, strategy='uniform')

# The less observations we have per interval, the harder to obtain a reliable probability vs positive fraction estimate. This, the functions seems more noisy.

plot_calibration_curve(y_test, probs, bins=5, strategy='uniform')

# **HOMEWORK 1**: Train another model, different from Logistic Regression and determine the calibration curve. Are probabilities calibrated?
#
# ## Imbalanced dataset

# +
X, y = create_dataset(weights=0.9)

# number of observations per class
y.value_counts()

# +
# balancing ratio

y.value_counts() / len(y)
# -

# ### Separate train and test

# +
# separate dataset into train and test

X_train, X_test, y_train, y_test = train_test_split(
    X,  
    y,  
    test_size=0.3,
    random_state=0)

X_train.shape, X_test.shape
# -

# ### Train Logistic Regression

# +
# Logistic Regression on imbalanced dataset

logit = LogisticRegression(
    penalty='l2',
    solver='newton-cg',
    C=1,
    random_state=0,
    max_iter=100,
).fit(X_train, y_train)

# score the test set
probs = logit.predict_proba(X_test)[:, 1]
# -

# ## Calibration Curve

plot_calibration_curve(y_test, probs, bins=10, strategy='uniform')

plot_calibration_curve(y_test, probs, bins=5, strategy='quantile')

plot_calibration_curve(y_test, probs, bins=4, strategy='quantile')

# The predictions do not seem to be balanced. Most likely it is because we do not have a sufficient number of observations of the positive class, to accurately determine the fraction of positives.
#
# **HOMEWORK 2**: increase the number of observations in the dataset, and if you want as well, the balancing ratio, and observe the change in the calibration curve. What do you see?



StarcoderdataJupyterScriptsDedupFiltered