# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# # Permutation Importance and Dealing with Correlated Features

# %load_ext watermark
# %watermark -a "<NAME>" -u -d -p numpy,pandas,matplotlib,sklearn,mlxtend

# %matplotlib inline
import matplotlib.pyplot as plt

# +
import numpy as np


np.random.seed(123)

y = np.zeros(1000)
y[:500] = 1

x1 = np.random.randn(1000)

x2 = np.empty(1000)
x2[:500] = np.random.randn(500)
x2[500:] = np.random.randn(500)+4

x3 = x2 

X = np.vstack((x3, x2, x1)).swapaxes(1, 0)
X.shape

# +
from sklearn.model_selection import train_test_split


X_train, X_test, y_train, y_test =\
    train_test_split(X, y, 
                     test_size=0.3, 
                     random_state=123, 
                     stratify=y)

# +
from sklearn.ensemble import RandomForestClassifier


forest = RandomForestClassifier(n_estimators=10,
                                random_state=123,
                                max_features=2)

forest.fit(X_train, y_train)

print('Training accuracy:', np.mean(forest.predict(X_train) == y_train)*100)
print('Test accuracy:', np.mean(forest.predict(X_test) == y_test)*100)
# -

corr = np.corrcoef(X_train.T)
corr.shape

# +
from mlxtend.plotting import heatmap


heatmap(corr, cmap='binary')
plt.savefig('5.pdf')
plt.show()

# +
from mlxtend.evaluate import feature_importance_permutation


imp_vals, imp_all = feature_importance_permutation(
    predict_method=forest.predict, 
    X=X_test,
    y=y_test,
    metric='accuracy',
    num_rounds=50,
    seed=123)

# +
std = np.std(imp_all, axis=1)
indices = np.argsort(imp_vals)[::-1]

plt.figure()
plt.title("Random Forest feature importance via permutation importance")
plt.bar(range(X_train.shape[1]), imp_vals[indices], yerr=std[indices])

plt.xticks(range(X_train.shape[1]), indices)
plt.xlim([-1, X_train.shape[1]])
plt.ylim([0, 0.6])

plt.tight_layout()
plt.savefig('6.pdf')
plt.show()

# +
X_train = X_train[:, 1:]
X_test = X_test[:, 1:]

forest = RandomForestClassifier(n_estimators=10,
                                random_state=123,
                                max_features=2)

forest.fit(X_train, y_train)

imp_vals, imp_all = feature_importance_permutation(
    predict_method=forest.predict, 
    X=X_test,
    y=y_test,
    metric='accuracy',
    num_rounds=50,
    seed=123)

std = np.std(imp_all, axis=1)
indices = np.argsort(imp_vals)[::-1]

plt.figure()
plt.title("Random Forest feature importance via permutation importance")
plt.bar(range(X_train.shape[1]), imp_vals[indices], yerr=std[indices])

plt.xticks(range(X_train.shape[1]), indices)
plt.xlim([-1, X_train.shape[1]])
plt.ylim([0, 0.6])

plt.tight_layout()
plt.savefig('7.pdf')
plt.show()
# -



StarcoderdataJupyterScriptsDedupFiltered