# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] id="7JfLUlawto_D"
# # 處理不平衡資料(imbalanced data)的一般訓練方法

# + [markdown] id="EyVFC9L_v_9J"
# 我們以kaggle-信用詐欺偵測為範例介紹，一般在處理資料類別(正例/負例樣本)高度不平衡的資料集如何訓練模型.

# + [markdown] id="mthoSGBAOoX-"
# 資料來源: [Credit Card Fraud Detection](https://www.kaggle.com/mlg-ulb/creditcardfraud) 
# 合計 284,807筆交易記錄(transactions)，其中有 492 異常交易(fraudulent transactions).
#
# **這個範例的異常交易是登記為正例樣本(positive samples)**
#
# 程式內容包括:
#
# * 載入資料集 (CSV file).
# * 建立訓練集及測試集..等資料.
# * 訓練baseline模型 (使用keras).
# * 評估模型績效 (precision and recall).
# * 一般處理 imbalanced data 的訓練手法:
#     * Class weighting
#     * Oversampling
#

# + [markdown] id="kRHmSyHxEIhN"
# ## 載入工具
# -

import warnings
warnings.filterwarnings("ignore")

# + id="JM7hDSNClfoK"
import tensorflow as tf
from tensorflow import keras

import os
import tempfile

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns

import sklearn
from sklearn.metrics import confusion_matrix, classification_report
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

# + id="c8o1FHzD-_y_"
mpl.rcParams['figure.figsize'] = (12, 10)
colors = plt.rcParams['axes.prop_cycle'].by_key()['color'] #繪圖用的顏色

# + [markdown] id="Z3iZVjziKHmX"
# ## 資料處理及資料探索(EDA)
# -

tf.__version__

# + id="pR_SnbMArXr7"
file = tf.keras.utils
#從google AIP data下載 CSV檔案資料
raw_df = pd.read_csv('datasets/creditcard.csv')
raw_df.head()

# + id="-fgdQgmwUFuj"
raw_df[['Time', 'V1', 'V2', 'V3', 'V4', 'V5', 'V6', 'Amount', 'Class']].describe()

# + [markdown] id="xWKB_CVZFLpB"
# ### 瞭解標記的類別資料不平衡狀況 (class label imbalance)

# + id="HCJFrtuY2iLF"
neg, pos = np.bincount(raw_df['Class'])
total = neg + pos
print('Examples:\n    Total: {}\n    Positive: {} ({:.2f}% of total)\n'.format(
    total, pos, 100 * pos / total))

# + [markdown] id="6qox6ryyzwdr"
# ### 資料清理
# 1.時間(`Time`): 這一欄位我們目前沒用到,故先去除.
#
# 2.數量(`Amount`): 這個欄位資料級距差異太大,取log讓它比較趨於線性可比較.

# + id="Ef42jTuxEjnj"
cleaned_df = raw_df.copy()

# 利用pop指令移除"Time"欄位
cleaned_df.pop('Time')

# "Amount"做log轉換, 轉換時加一個很小的數值,以免轉到 np.log(0) = -inf
eps = 0.0001
cleaned_df['Log Ammount'] = np.log(cleaned_df.pop('Amount')+eps)

# + [markdown] id="uSNgdQFFFQ6u"
# ### 切分資料集: 訓練集(train) 測試集(test) 驗證集(val, from training set)

# + id="xfxhKg7Yr1-b"
# 使用 sklearn API來切分資料集.
train_df, test_df = train_test_split(cleaned_df, test_size=0.2)
train_df, val_df = train_test_split(train_df, test_size=0.2)

# 整理 label資料.
train_labels = np.array(train_df.pop('Class'))
bool_train_labels = train_labels != 0
val_labels = np.array(val_df.pop('Class'))
test_labels = np.array(test_df.pop('Class'))

train_features = np.array(train_df)
val_features = np.array(val_df)
test_features = np.array(test_df)
# -

print("訓練集中的正例樣本(class=1)數量: ",len(bool_train_labels[bool_train_labels == True]))

# + [markdown] id="8a_Z_kBmr7Oh"
# ### 正規化
# 利用sklearn StandardScaler (mean=0, standard deviation=1)
# 把資料集的特徵值域用標準差做縮放.
#
# (並利用np.clip裁切值域範例在-5 ~ +5之間)
# -

np.clip(-15,-5,5), np.clip(99,-5,5)

# + id="IO-qEUmJ5JQg"
scaler = StandardScaler()

train_features = scaler.fit_transform(train_features)
val_features = scaler.transform(val_features)
test_features = scaler.transform(test_features)

#把值域裁切在 -5 ~ 5之間:
train_features = np.clip(train_features, -5, 5)
val_features = np.clip(val_features, -5, 5)
test_features = np.clip(test_features, -5, 5)

print('Training labels shape:', train_labels.shape)
print('Validation labels shape:', val_labels.shape)
print('Test labels shape:', test_labels.shape)

print('Training features shape:', train_features.shape)
print('Validation features shape:', val_features.shape)
print('Test features shape:', test_features.shape)

# + [markdown] id="uQ7m9nqDC3W6"
# ### 觀查資料分佈狀況
#
# 比較兩組欄位資料的數值分佈狀況. **Positive (fraudulent transactions) vs. Negative (normal transactions)**

# + id="raK7hyjd_vf6"
pos_df = pd.DataFrame(train_features[ bool_train_labels], columns=train_df.columns)
neg_df = pd.DataFrame(train_features[~bool_train_labels], columns=train_df.columns)

sns.jointplot(pos_df['V5'], pos_df['V6'], kind='hex', xlim=(-5,5), ylim=(-5,5))
plt.suptitle("Positive distribution (fraudulent transactions)")

sns.jointplot(neg_df['V5'], neg_df['V6'], kind='hex', xlim=(-5,5), ylim=(-5,5))
_ = plt.suptitle("Negative distribution (normal transactions)")

# + [markdown] id="qFK1u4JX16D8"
# ## 定義模型結構及使用的評估指標
# -

METRICS = [
      #keras.metrics.TruePositives(name='tp'),
      #keras.metrics.FalsePositives(name='fp'),
      #keras.metrics.TrueNegatives(name='tn'),
      #keras.metrics.FalseNegatives(name='fn'), 
      keras.metrics.BinaryAccuracy(name='accuracy'),
      keras.metrics.Precision(name='precision'),
      keras.metrics.Recall(name='recall'),
      keras.metrics.AUC(name='auc'),
]


# + id="3JQDzUqT3UYG"
def make_model(metrics=METRICS, output_bias=None):
    if output_bias is not None:
        output_bias = tf.keras.initializers.Constant(output_bias)

    model = keras.Sequential([
              keras.layers.Dense(64, activation='relu',input_shape=(train_features.shape[-1],)),
              keras.layers.Dropout(0.5),
              keras.layers.Dense(32, activation='relu'),
              keras.layers.Dropout(0.5),
              keras.layers.Dense(1, activation='sigmoid', bias_initializer=output_bias),
            ])
    return model


# + [markdown] id="SU0GX6E6mieP"
# ### 評估指標 (metrics)
# 指標公式介紹:
# https://stanford.edu/~shervine/l/zh-tw/teaching/cs-229/cheatsheet-machine-learning-tips-and-tricks
# *   **positives** - 正例樣本
# *   **negatives** - 負例樣本
# *   predict **False** - 預測錯誤(**incorrectly** classified)
# *   predict **True** - 預測正確(**correctly** classified)
# *   準確度(**Accuracy**) - 在所有預測樣本中，預測正確的比率
# >   $\frac{\text{true samples}}{\text{total samples}}$
# *   精密度(**Precision**) - 所有正例樣本中，預測正確的比率
# >   $\frac{\text{true positives}}{\text{true positives + false positives}}$
# *   召回率(**Recall**) - 所有預測樣本中，判別正例樣本且預測正確的比率
# >   $\frac{\text{true positives}}{\text{true positives + false negatives}}$
# *   **AUC** - Receiver Operating Characteristic curve (ROC)曲線所覆蓋的面積，數值愈接近1愈好.
#
# Note: 在資料類別不平衡的任務，準確度(Accuracy)不是一個好的衡量指標，你可能在模型準確率99.8%+ accuracy 的狀況下，預測不出正確的分類結果。  
#
# -

# ----

# + [markdown] id="FYdhSAoaF_TK"
# ## 建立及訓練 Baseline model

# + id="1xlR_dekzw7C"
model = make_model()
model.compile( optimizer=keras.optimizers.Adam(lr=0.001),
               loss=keras.losses.BinaryCrossentropy(),
               metrics=METRICS)

model.summary()

# + id="ouUkwPcGQsy3"
early_stopping = tf.keras.callbacks.EarlyStopping(
    monitor='val_auc', 
    verbose=1,
    patience=5,
    mode='max',
    restore_best_weights=True)
# -

EPOCHS = 100
BATCH_SIZE = 1024
history = model.fit(train_df, train_labels,
              batch_size=BATCH_SIZE,
              epochs=EPOCHS,
              verbose=1,
              shuffle=True,
              validation_split=0.2, callbacks=[early_stopping]) # 加上callbacks:earlystopping

# +
''' Visualize the training/validation loss and training/validation accuracy'''
plt.rcParams['figure.figsize'] = 10, 7

loss = history.history.get('loss')       # 取出訓練完成後loss的訓練數據
val_loss = history.history.get('val_loss')   # 取出訓練完成後val_loss的訓練數據
auc = history.history.get('auc')
val_auc = history.history.get('val_auc')

plt.figure(0)
plt.subplot(121)

# 個別繪出loss以及val_loss的線條
plt.plot(range(len(loss)), loss, label='Training Loss')
plt.plot(range(len(val_loss)), val_loss, label='Validation Loss')
plt.legend(loc='best')
plt.title('Loss')

plt.subplot(122)
# 個別繪出auc以及val_auc的線條
plt.plot(range(len(auc)), auc, label='Training AUC')
plt.plot(range(len(val_auc)), val_auc, label='Validation AUC')
plt.legend(loc='best')
plt.title('AUC')

plt.show()

# + id="Wx7ND3_SqckO"
pred = model.evaluate(val_df, val_labels, verbose=2)

# + id="LopSd-yQqO3a"
model.predict_classes(test_features[0:1])[0], model.predict(test_features[0:1])[0]
# -

# ### 儲存 initial weights (baseline model)

initial_weights = os.path.join(tempfile.mkdtemp(), 'initial_weights')
model.save_weights(initial_weights)

initial_weights


# + id="WTSkhT1jyGu6"
def plot_metrics(history):
    metrics = ['loss', 'auc', 'precision', 'recall']
    for n, metric in enumerate(metrics):
        name = metric.replace("_"," ").capitalize()
        plt.subplot(2,2,n+1)
        plt.plot(history.epoch, history.history[metric], color=colors[0], label='Train')
        plt.plot(history.epoch, history.history['val_'+metric],
                 color=colors[0], linestyle="--", label='Val')
        plt.xlabel('Epoch')
        plt.ylabel(name)
    if metric == 'loss':
        plt.ylim([0, plt.ylim()[1]])
    elif metric == 'auc':
        plt.ylim([0.8,1])
    else:
        plt.ylim([0,1])

    plt.legend()


# + id="u6LReDsqlZlk"
#儲存的第一版 base model的訓練狀況
plot_metrics(history)

# + [markdown] id="aJC1booryouo"
# ### 評估指標(metrics)

# + id="aNS796IJKrev"
train_predictions_baseline = model.predict(train_features, batch_size=BATCH_SIZE)
test_predictions_baseline = model.predict(test_features, batch_size=BATCH_SIZE)
# -

#使用 sciki-learn API的分類報告
test_pred = model.predict_classes(test_features, batch_size=BATCH_SIZE)
print(classification_report(test_labels, test_pred))


# + id="MVWBGfADwbWI"
#混淆矩陣作成圖形顯示
def plot_cm(labels, predictions, p=0.5):
    cm = confusion_matrix(labels, predictions > p)
    plt.figure(figsize=(5,5))
    sns.heatmap(cm, annot=True, fmt="d")
    plt.title('Confusion matrix @{:.2f}'.format(p))
    plt.ylabel('Actual label')
    plt.xlabel('Predicted label')

    print('Legitimate Transactions Detected (True Negatives): ', cm[0][0])
    print('Legitimate Transactions Incorrectly Detected (False Positives): ', cm[0][1])
    print('Fraudulent Transactions Missed (False Negatives): ', cm[1][0])
    print('Fraudulent Transactions Detected (True Positives): ', cm[1][1])
    print('Total Fraudulent Transactions: ', np.sum(cm[1]))


# + [markdown] id="nOTjD5Z5Wp1U"
# ### 利用 `model.Evaluate` 方法，計算模型績效

# + id="poh_hZngt2_9"
baseline_results = model.evaluate(test_features, test_labels,
                                  batch_size=BATCH_SIZE, verbose=0)
for name, value in zip(model.metrics_names, baseline_results):
    print(name, ': ', value)
print()

plot_cm(test_labels, test_predictions_baseline)


# + [markdown] id="P-QpQsip_F2Q"
# ### 繪制ROC圖表

# + id="lhaxsLSvANF9"
def plot_roc(name, labels, predictions, **kwargs):
    fp, tp, _ = sklearn.metrics.roc_curve(labels, predictions)

    plt.plot(100*fp, 100*tp, label=name, linewidth=2, **kwargs)
    plt.xlabel('False positives [%]')
    plt.ylabel('True positives [%]')
    plt.xlim([-0.5,20])
    plt.ylim([80,100.5])
    plt.grid(True)
    ax = plt.gca()
    ax.set_aspect('equal')


# + id="DfHHspttKJE0"
plot_roc("Train Baseline", train_labels, train_predictions_baseline, color=colors[0])
plot_roc("Test Baseline", test_labels, test_predictions_baseline, color=colors[0], linestyle='--')
plt.legend(loc='lower right')
# -

# ----

# ## 一般處理 imbalanced data 的訓練手法

# + [markdown] id="cveQoiMyGQCo"
# ### 類別權重 (Class weights)

# + [markdown] id="ePGp6GUE1WfH"
# ### 設算類別權重
#
# 我們需合理設算樣本數少的類別在模型訓練時要加重它的權重，讓模型呈現上能有更好表現(好的績效指標).
# -

print("目前資料類別的筆數狀況: 負例樣本(class-0): {} 筆, 正例樣本(class-1): {} 筆".format(neg, pos))

# + id="qjGWErngGny7"
# 基本分兩類時，我們期忘樣本比例為 total/2 ，各取一半
# 再依各類別資料比數(倒數)做加權, 讓筆數多的資料權重較少, 筆數少的資料權重較大
weight_for_0 = (1 / neg)*(total)/2.0 
weight_for_1 = (1 / pos)*(total)/2.0

class_weight = {0: weight_for_0, 1: weight_for_1}

print('class 0 (negative)權重比率: {:.2f}'.format(weight_for_0))
print('class 1 (positive)權重比率: {:.2f}'.format(weight_for_1))

# + [markdown] id="Mk1OOE2ZSHzy"
# ### Train a model with class weights
#
# 我們利用 `model.fit()` 參數: `class_weight` 來設定這次模型訓練的類別權重
#
# **class_weight**
# ```
# Optional dictionary mapping class indices (integers) to a weight (float) value, used for weighting the loss function (during training only). This can be useful to tell the model to "pay more attention" to samples from an under-represented class.
# ```

# + id="UJ589fn8ST3x"
weighted_model = make_model()
weighted_model.load_weights(initial_weights)

weighted_model.compile( optimizer=keras.optimizers.Adam(lr=0.001),
               loss=keras.losses.BinaryCrossentropy(),
               metrics=METRICS)

weighted_history = weighted_model.fit(
    train_features,
    train_labels,
    batch_size=BATCH_SIZE,
    epochs=EPOCHS,
    callbacks=[early_stopping],
    validation_data=(val_features, val_labels),
    #設定 class weights 參數:
    class_weight=class_weight)

# + [markdown] id="R0ynYRO0G3Lx"
# ### 模型訓練狀況

# + id="BBe9FMO5ucTC"
plot_metrics(weighted_history)

# + [markdown] id="REy6WClTZIwQ"
# ### 評估指標

# + id="nifqscPGw-5w"
train_predictions_weighted = weighted_model.predict(train_features, batch_size=BATCH_SIZE)
test_predictions_weighted = weighted_model.predict(test_features, batch_size=BATCH_SIZE)
# -

test_pred = weighted_model.predict_classes(test_features, batch_size=BATCH_SIZE)
print(classification_report(test_labels, test_pred))

# + id="owKL2vdMBJr6"
weighted_results = weighted_model.evaluate(test_features, test_labels,
                                           batch_size=BATCH_SIZE, verbose=0)
for name, value in zip(weighted_model.metrics_names, weighted_results):
    print(name, ': ', value)
print()

plot_cm(test_labels, test_predictions_weighted)

# + [markdown] id="hXDAwyr0HYdX"
# ### 繪制ROC圖表

# + id="3hzScIVZS1Xm"
plot_roc("Train Baseline", train_labels, train_predictions_baseline, color=colors[0])
plot_roc("Test Baseline", test_labels, test_predictions_baseline, color=colors[0], linestyle='--')

plot_roc("Train Weighted", train_labels, train_predictions_weighted, color=colors[1])
plot_roc("Test Weighted", test_labels, test_predictions_weighted, color=colors[1], linestyle='--')

plt.legend(loc='lower right')
# -

# ----

# + [markdown] id="5ysRtr6xHnXP"
# ## Oversampling

# + [markdown] id="18VUHNc-UF5w"
# 利用重新抽樣，增加抽樣那些樣比數較少的類別.
# -

print("(training set總筆數, train-正例樣本數, train-負例樣本數 )")
len(bool_train_labels), len(bool_train_labels[bool_train_labels == True]), len(bool_train_labels[bool_train_labels == False])

# + id="sHirNp6u7OWp"
#從training set中分出正例及負例樣本數, 及其label資料
pos_features = train_features[bool_train_labels]
neg_features = train_features[~bool_train_labels]

pos_labels = train_labels[bool_train_labels]
neg_labels = train_labels[~bool_train_labels]

# + [markdown] id="WgBVbX7P7QrL"
# #### 正例樣本 oversampling
# 從正例樣本(訓練)資料集，oversampling 跟負例樣本(訓練)資料集相同的筆數. 
#
# #### 方法一: 使用 `np.random.choice` (*這方法不確保訓練過程也能均勻取樣)

# + id="BUzGjSkwqT88"
ids = np.arange(len(pos_features))
choices = np.random.choice(ids, len(neg_features))

res_pos_features = pos_features[choices]
res_pos_labels = pos_labels[choices]

res_pos_features.shape

# + id="7ie_FFet6cep"
resampled_features = np.concatenate([res_pos_features, neg_features], axis=0)
resampled_labels = np.concatenate([res_pos_labels, neg_labels], axis=0)

order = np.arange(len(resampled_labels))
np.random.shuffle(order)
resampled_features = resampled_features[order]
resampled_labels = resampled_labels[order]

print("oversampling後的訓練資料集之筆數及欄位數目:")
resampled_features.shape

# + [markdown] id="IYfJe2Kc-FAz"
# #### 方法二: 使用 `tf.data` 工具 (可以做到訓練過程也保持均勻取樣)

# + [markdown] id="usyixaST8v5P"
# 使用`tf.data`工具的重覆取樣(`repeat`)功能來進行oversampling.

# + id="yF4OZ-rI6xb6"
BUFFER_SIZE = 100000

def make_ds(features, labels):
    ds = tf.data.Dataset.from_tensor_slices((features, labels))#.cache()
    ds = ds.shuffle(BUFFER_SIZE).repeat() #在buffer size大小的樣本中重覆取樣
    return ds

pos_ds = make_ds(pos_features, pos_labels)
neg_ds = make_ds(neg_features, neg_labels)
# -

type(pos_ds)

# + [markdown] id="RNQUx-OA-oJc"
# RepeatDataset 可利用take()來取出(feature, label)成對的資料

# + id="llXc9rNH7Fbz"
for features, label in pos_ds.take(1):
    print("Features:\n", features.numpy())
    print()
    print("Label: ", label.numpy())

# + [markdown] id="sLEfjZO0-vbN"
# 使用`experimental.sample_from_datasets`來合併 pos_ds 及 neg_ds，同時指定它們的合併比例(參數: weights)

# + id="e7w9UQPT9wzE"
resampled_ds = tf.data.experimental.sample_from_datasets([pos_ds, neg_ds], weights=[0.5, 0.5])
resampled_ds = resampled_ds.batch(BATCH_SIZE).prefetch(2)

# + id="EWXARdTdAuQK"
for features, label in resampled_ds.take(1):
    print("BATCH_SIZE:", BATCH_SIZE)
    print("Features shape:", features.shape)
    print("Label shape: ", label.shape)
    print("each BATCH neg vs. pos 大約的數量:",(len(label[label==0]),len(label[label==1])))

# + [markdown] id="irgqf3YxAyN0"
# 需事先設算稍後訓練過程每一個epoch要分幾個step來更新模型(每個step使用BATCH_SIZE的取樣數量)
# -

(neg, pos)

# + id="xH-7K46AAxpq"
resampled_steps_per_epoch = np.ceil(2.0*neg/BATCH_SIZE) #因為neg vs. pos大約是各半, 故以 2倍neg的樣本數來大略估算
resampled_steps_per_epoch

# + [markdown] id="XZ1BvEpcBVHP"
# ### 使用oversampled data來訓練 (且每一batch取樣大略都均勻取樣)

# + id="soRQ89JYqd6b"
resampled_model = make_model()
resampled_model.load_weights(initial_weights)

resampled_model.compile( optimizer=keras.optimizers.Adam(lr=0.001),
               loss=keras.losses.BinaryCrossentropy(),
               metrics=METRICS)

val_ds = tf.data.Dataset.from_tensor_slices((val_features, val_labels)).cache()
val_ds = val_ds.batch(BATCH_SIZE).prefetch(2) 

resampled_history = resampled_model.fit(
    resampled_ds,
    epochs=EPOCHS,
    steps_per_epoch=resampled_steps_per_epoch,
    callbacks=[early_stopping],
    validation_data=val_ds)

# + [markdown] id="klHZ0HV76VC5"
# ### 模型訓練狀況

# + id="YoUGfr1vuivl"
plot_metrics(resampled_history)

# + [markdown] id="bUuE5HOWZiwP"
# ### 評估指標

# + id="C0fmHSgXxFdW"
train_predictions_resampled = resampled_model.predict(train_features, batch_size=BATCH_SIZE)
test_predictions_resampled = resampled_model.predict(test_features, batch_size=BATCH_SIZE)
# -

test_pred = resampled_model.predict_classes(test_features, batch_size=BATCH_SIZE)
print(classification_report(test_labels, test_pred))

# + id="FO0mMOYUDWFk"
resampled_results = resampled_model.evaluate(test_features, test_labels,
                                             batch_size=BATCH_SIZE, verbose=0)
for name, value in zip(resampled_model.metrics_names, resampled_results):
    print(name, ': ', value)
print()

plot_cm(test_labels, test_predictions_resampled)

# + [markdown] id="_xYozM1IIITq"
# ### 繪制ROC圖表

# + id="fye_CiuYrZ1U"
plot_roc("Train Baseline", train_labels, train_predictions_baseline, color=colors[0])
plot_roc("Test Baseline", test_labels, test_predictions_baseline, color=colors[0], linestyle='--')

plot_roc("Train Weighted", train_labels, train_predictions_weighted, color=colors[1])
plot_roc("Test Weighted", test_labels, test_predictions_weighted, color=colors[1], linestyle='--')

plot_roc("Train Resampled", train_labels, train_predictions_resampled, color=colors[2])
plot_roc("Test Resampled", test_labels, test_predictions_resampled, color=colors[2], linestyle='--')
plt.legend(loc='lower right')
# -

# ----



StarcoderdataJupyterScriptsDedupFiltered