# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# + [markdown] colab_type="text" id="ncGMi7XXo-8g"
# # Kapitel 5: Featureauswahl

# + colab={} colab_type="code" id="tum7pL55o-8h"
import warnings
warnings.filterwarnings('ignore')

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" id="qPUJu4mOo-8k" outputId="66cb69b1-1d62-45cc-d337-6c571dae9e06"
# %matplotlib inline
# %pylab inline

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" id="iQF9mgcYo-8q" outputId="240588fa-9695-49d5-85c8-a11204421ef4"
import sklearn
print(sklearn.__version__)

# + colab={} colab_type="code" id="DfcQU94uo-8u"
import numpy as np

# + colab={} colab_type="code" id="cjffiT9Po-8x"
import matplotlib.pyplot as plt

# + [markdown] colab_type="text" id="p5HliKB5o-8z"
# ## Fluch der hohen Dimensionen

# + colab={} colab_type="code" id="HNaFi456o-8z"
n = 100
vmin = 0; vmax = 10
x1 =  np.random.uniform(vmin, vmax, n)
x2 =  np.random.uniform(vmin, vmax, n)
x3 =  np.random.uniform(vmin, vmax, n)

# + colab={} colab_type="code" id="gvuboZwlo-82"
# #plt.hist?

# + colab={"base_uri": "https://localhost:8080/", "height": 660} colab_type="code" id="VkIb2Y-Go-84" outputId="b1d8c08b-fc70-4b47-8a49-a9be56e60717"
# Eine Dimension

fig = plt.figure(figsize=(16, 11))
ax = fig.add_subplot(111)

ax.hist(x1, alpha=0.6, edgecolor='black', lw=1, bins=np.arange(0, 11, 1))

ax.set_xlabel('X1')
ax.set_ylabel('n samples')

# fig.savefig('ML_0512.png', bbox_inches='tight')
plt.show()

# + colab={"base_uri": "https://localhost:8080/", "height": 660} colab_type="code" id="eUKEnZPfo-86" outputId="6d118081-e3c7-44fc-deb8-f9f3129242b1"
# Zwei Dimensionen

fig = plt.figure(figsize=(16, 11))
ax = fig.add_subplot(111)

ax.scatter(x1, x2, c="b", marker="o")

ax.set_xlabel('X1')
ax.set_ylabel('X2')
ax.set_xticks(np.arange(0, 11, 1) )
ax.set_yticks(np.arange(0, 11, 1) )
ax.grid(color='k', linestyle='-', linewidth=1, alpha=0.6)

# fig.savefig('ML_0513.png', bbox_inches='tight')
plt.show()
# plt.clf()

# + colab={"base_uri": "https://localhost:8080/", "height": 629} colab_type="code" id="wmk2Mw5Uo-88" outputId="e3ba42bc-8fe4-46d6-c20d-dd30436bde98"
# Drei Dimensionen
from mpl_toolkits.mplot3d import Axes3D
fig = plt.figure(figsize=(16, 11))
ax = fig.add_subplot(111, projection='3d')

ax.scatter(x1, x2, x3, c="b", marker="o")

# ax.plot_wireframe((3,4,4,3,3, 3,4,4,3,3, 3,3,4,4,4,4),
#                   (5,5,6,6,5, 5,5,6,6,5, 6,6,6,6,5,5),
#                   (2,2,2,2,2, 3,3,3,3,3, 3,2,2,3,3,2),
#                   color='r', rstride=1, cstride=1, alpha=0.9)

ax.set_xticks(np.arange(0, 11, 1) )
ax.set_yticks(np.arange(0, 11, 1) )
ax.set_zticks(np.arange(0, 11, 1) )
ax.grid(color='k', linestyle='-', linewidth=1, alpha=0.6)
ax.set_xlabel('X1')
ax.set_ylabel('X2')
ax.set_zlabel('X3')

# fig.savefig('ML_0514.png', bbox_inches='tight')
plt.show()
# plt.clf()

# + [markdown] colab_type="text" id="oJ0D3ChGo-8_"
# ## Overfitting und Underfitting: Model-Komplexität vs Datenmenge

# + colab={} colab_type="code" id="6rBG3mrWo-8_"
np.random.RandomState(1)

n_samples = 20
X = np.random.uniform(-2, 2, n_samples)
y = X**3 + np.random.uniform(0, 2, n_samples)

# + colab={"base_uri": "https://localhost:8080/", "height": 678} colab_type="code" id="If_D49vDo-9D" outputId="d4a79d14-f448-4d06-f8d2-594b23dfad87"
fig, ax = plt.subplots(figsize=(11, 11))

print(X.shape, y.shape)
plt.scatter(X, y, color='navy', s=30, marker='o')

plt.xlabel('x')
plt.ylabel('y')

# fig.savefig('ML_0504.png', bbox_inches='tight')
plt.show()
# plt.clf()

# + colab={"base_uri": "https://localhost:8080/", "height": 660} colab_type="code" id="cVljjox3o-9F" outputId="b8eb6bc8-836f-4d2e-e4f9-6623256fc28e"
from sklearn.pipeline import make_pipeline
from sklearn.linear_model import Ridge, LinearRegression
from sklearn.preprocessing import PolynomialFeatures

fig, ax = plt.subplots(figsize=(11, 11))
plt.scatter(X, y, color='navy', s=30, marker='o')

x_plot = np.linspace(-2, 2, 100)
poly_model = make_pipeline(PolynomialFeatures(3),
                           LinearRegression())
poly_model.fit(X[:, np.newaxis], y)
y_plot = poly_model.predict(x_plot[:, np.newaxis])
plt.plot(x_plot, y_plot, lw=2, color="red")
plt.ylim(-12, 12)

plt.xlabel('x')
plt.ylabel('y')

# fig.savefig('ML_0505.png', bbox_inches='tight')
plt.show()
# plt.clf()

# + colab={"base_uri": "https://localhost:8080/", "height": 660} colab_type="code" id="qx9d0Lg1o-9H" outputId="14de6319-8e3c-4046-84ed-f143e14411c1"
fig, ax = plt.subplots(figsize=(11, 11))
plt.scatter(X, y, color='navy', s=30, marker='o')


poly_model = make_pipeline(PolynomialFeatures(1),
                           LinearRegression())
poly_model.fit(X[:, np.newaxis], y)
y_plot = poly_model.predict(x_plot[:, np.newaxis])
plt.plot(x_plot, y_plot, lw=2, color="red")
plt.ylim(-9, 9)

plt.xlabel('x')
plt.ylabel('y')

# fig.savefig('ML_0507.png', bbox_inches='tight')
plt.show()
# plt.clf()

# + colab={"base_uri": "https://localhost:8080/", "height": 664} colab_type="code" id="m9gyCATFo-9J" outputId="1880e461-b905-45bd-abf1-a8091c959f27"
fig, ax = plt.subplots(figsize=(11, 11))
plt.scatter(X, y, color='navy', s=30, marker='o')

poly_model = make_pipeline(PolynomialFeatures(20), LinearRegression())
poly_model.fit(X[:, np.newaxis], y)
y_plot = poly_model.predict(x_plot[:, np.newaxis])
plt.plot(x_plot, y_plot, lw=2, color="red")
plt.ylim(-10, 10)

plt.xlabel('x')
plt.ylabel('y')

# fig.savefig('ML_0506.png', bbox_inches='tight')
plt.show()
# plt.clf()

# + [markdown] colab_type="text" id="2VeSYK7qo-9K"
# ### Mehr Datensätze

# + colab={} colab_type="code" id="T6d_sBJTo-9L"
n_samples = 200
X = np.random.uniform(-2, 2, n_samples)
y = X**3 + np.random.uniform(0, 2, n_samples)

# + colab={"base_uri": "https://localhost:8080/", "height": 296} colab_type="code" id="tnEhZ8qIo-9O" outputId="816a1831-4c17-41eb-cbc8-35140202af52"
print(X.shape, y.shape)
plt.scatter(X, y, color='navy', s=30, marker='o', label="training points")

plt.xlabel('x')
plt.ylabel('y')

# fig.savefig('ML_0508.png', bbox_inches='tight')
plt.show()
# plt.clf()

# + colab={"base_uri": "https://localhost:8080/", "height": 660} colab_type="code" id="d9sBpbCpo-9R" outputId="cf69de1d-b147-4332-abb5-64d6aa678ca8"
fig, ax = plt.subplots(figsize=(11, 11))
plt.scatter(X, y, color='navy', s=30, marker='o', label="training points")

poly_model = make_pipeline(PolynomialFeatures(3),
                           LinearRegression())
poly_model.fit(X[:, np.newaxis], y)
y_plot = poly_model.predict(x_plot[:, np.newaxis])
plt.plot(x_plot, y_plot, lw=2, color="red")
plt.ylim(-12, 12)

plt.xlabel('x')
plt.ylabel('y')

# fig.savefig('ML_0509.png', bbox_inches='tight')
plt.show()
# plt.clf()

# + colab={"base_uri": "https://localhost:8080/", "height": 664} colab_type="code" id="dvzRCcsCo-9T" outputId="677cacbc-6b3f-490f-90c2-9cfc69704775"
fig, ax = plt.subplots(figsize=(11, 11))
plt.scatter(X, y, color='navy', s=30, marker='o', label="training points")

poly_model = make_pipeline(PolynomialFeatures(20),
                           LinearRegression())
poly_model.fit(X[:, np.newaxis], y)
y_plot = poly_model.predict(x_plot[:, np.newaxis])
plt.plot(x_plot, y_plot, lw=2, color="red")
plt.ylim(-8, 8)

plt.xlabel('x')
plt.ylabel('y')

# fig.savefig('ML_0510.png', bbox_inches='tight')
plt.show()
# plt.clf()

# + colab={"base_uri": "https://localhost:8080/", "height": 660} colab_type="code" id="B9dJdby6o-9V" outputId="70a3e065-b6de-42c1-9fb7-31bd45a4d35e"
fig, ax = plt.subplots(figsize=(11, 11))
plt.scatter(X, y, color='navy', s=30, marker='o', label="training points")

poly_model = make_pipeline(PolynomialFeatures(1),
                           LinearRegression())
poly_model.fit(X[:, np.newaxis], y)
y_plot = poly_model.predict(x_plot[:, np.newaxis])
plt.plot(x_plot, y_plot, lw=2, color="red")
plt.ylim(-9, 9)

plt.xlabel('x')
plt.ylabel('y')

# fig.savefig('ML_0511.png', bbox_inches='tight')
plt.show()
# plt.clf()

# + [markdown] colab_type="text" id="SGNT--o_o-9Y"
# ## Univariate Feature Exploration

# + colab={"base_uri": "https://localhost:8080/", "height": 51} colab_type="code" id="8mh52lDYo-9Y" outputId="5e31f1c4-c451-40cc-e0a4-4ece05f28ea9"
from sklearn.datasets import load_iris
from sklearn.svm import SVC

iris = load_iris()

X = iris.data
y = iris.target
print(X.shape, y.shape)

import numpy as np
import matplotlib.pyplot as plt

svc = SVC(kernel='linear', C=1E0)
print(X.shape, y.shape)

n_classes = 3
colors = 'byr'
CMAP = colors
plot_step = 0.01

# Add random noise
rns = np.random.RandomState(12)
#noise1 = rns.lognormal(mean=1, size=(len(X), 1))
noise2 = rns.uniform(0, 6, size=(len(X), 1))

#X_noise = np.hstack([X, noise1])
X_noise = np.hstack([X, noise2])

# + colab={"base_uri": "https://localhost:8080/", "height": 755} colab_type="code" id="fcMKN8vZo-9a" outputId="94cbe773-9315-43d7-fea7-3256d2052239"
Y_feature_names = iris.feature_names
Y_target_names = iris.target_names
Y_feature_names = np.append(Y_feature_names, 'noise1')
#Y_feature_names = np.append(Y_feature_names, 'noise2')
Y_target_names = np.append(Y_target_names, 'noise1')
#Y_target_names = np.append(Y_target_names, 'noise2')


#fig = plt.figure(1, figsize=(9, 16))
fig = plt.figure(1, figsize=(16, 9))

BINS = []
BINS.append(np.arange(4, 8, 0.1))
BINS.append(np.arange(2, 5, 0.1))
BINS.append(np.arange(1, 7, 0.1))
BINS.append(np.arange(0, 3, 0.1))
BINS.append(np.arange(0, 6, 0.1))
#BINS.append(np.arange(0, 6, 0.1))
for fid in range(4):
#for fid in range(5):
    X = X_noise[:, fid]
    y = iris.target

    #plt.subplot(3, 2, fid + 1)
    plt.subplot(2, 2, fid + 1)

    plt.xlabel(Y_feature_names[fid])
    plt.ylabel('n examples')
    plt.axis("tight")

    for i, color in zip(range(n_classes), colors):
        idx = np.where(y == i)
        clf = svc.fit(X.reshape([150,1]), y)
        print(clf.score(X.reshape([150,1]), y))
        plt.hist(X[idx], alpha=0.6, color=color, edgecolor='black', lw=1, label=Y_target_names[i],
                 bins=BINS[fid])
        if fid==3:
            plt.legend(loc='upper right')

    plt.axis("tight")

plt.show()

# fig.savefig('ML_0501.png', bbox_inches='tight')
# plt.clf()

# + [markdown] colab_type="text" id="hPXEwCqyo-9b"
# ## Bivariate Feature Exploration

# + colab={"base_uri": "https://localhost:8080/", "height": 553} colab_type="code" id="cCdZA0VXo-9b" outputId="a1315ff2-af3d-4abd-fcd6-7139c127cbdf"
from scipy.stats import pearsonr

Y_feature_names = iris.feature_names
#Y_target_names = iris.target_names
#Y_feature_names = np.append(Y_feature_names, 'noise1')
#Y_feature_names = np.append(Y_feature_names, 'noise2')
#Y_target_names = np.append(Y_target_names, 'noise1')
#Y_target_names = np.append(Y_target_names, 'noise2')


n_classes = 3
colors = 'byr'
CMAP = colors
plot_step = 0.01

#____________________________________________________________________
fig = plt.figure(1, figsize=(18, 9))

pos = [[6.2, 4.2], [4.5, 6.5], [7, 0.5], [3.5, 3], [3.5, 1], [5, 0.5]]

for pairidx, pair in enumerate([[0, 1], [0, 2], [0, 3],
                                [1, 2], [1, 3], [2, 3]]):
    X = iris.data[:, pair]
    y = iris.target

    plt.subplot(2, 3, pairidx + 1)

    plt.xlabel(iris.feature_names[pair[0]])
    plt.ylabel(iris.feature_names[pair[1]])
    plt.axis("tight")

    for i, color in zip(range(n_classes), colors):
        idx = np.where(y == i)
        plt.scatter(X[idx, 0], X[idx, 1], c=color, edgecolor='black', lw=2, label=iris.target_names[i],
                    cmap=CMAP)

    r = "r = " + str(round(pearsonr(X[:, 0], X[:, 1])[0], 3))
    plt.text(pos[pairidx][0], pos[pairidx][1], r)
    plt.axis("tight")
    plt.axis("tight")

plt.legend(loc='upper left')
plt.show()

# fig.savefig('ML_0502.png', bbox_inches='tight')
# plt.clf()

# + [markdown] colab_type="text" id="YFFZBNy-o-9e"
# ## Korrelation zwischen Feature und Target

# + colab={"base_uri": "https://localhost:8080/", "height": 85} colab_type="code" id="vKuVNpILo-9f" outputId="c56b8185-809f-4085-f9ea-8de88b9576f8"
from sklearn.datasets import load_iris
import numpy as np
from scipy.stats import pearsonr # pearson package from scipy
iris = load_iris() # reload data
X = iris.data
y = iris.target
for fid in (0, 1, 2, 3): # loop over all features
    idx = np.where( (y == 0) | (y == 1) )
    x = X[idx]
    x = x[:, fid]
    print(iris.feature_names[fid], pearsonr(x, y[idx])[0])

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" id="D7Bfdkw6o-9i" outputId="555238c6-a7e5-4823-9ee2-42bcbf26d0cb"
x = np.random.uniform(-1, 1, 1000)
print(pearsonr(x, x**2)[0])

# + [markdown] colab_type="text" id="fl690nJfo-9j"
# ## Principal Component Analyse

# + colab={"base_uri": "https://localhost:8080/", "height": 136} colab_type="code" id="JwLnd5Kqo-9j" outputId="e480faa7-addc-400e-d42a-ad5e2ffb9ea9"
import numpy as np
import matplotlib.pyplot as plt

rns = np.random.RandomState(12)

size = 200
X = np.zeros((size, 2))
x1 = rns.uniform(0, 2, size=size)
x2 = -1.1*x1+1.8 + rns.normal(0, 0.2, size=size)
X[:, 0] = x1
X[:, 1] = x2

from sklearn.decomposition import PCA

pca = PCA(n_components=2, whiten=True)
pca.fit(X)

print(pca.explained_variance_)
print()
print(pca.components_)
print()
print(pca.mean_)
print()

# + colab={"base_uri": "https://localhost:8080/", "height": 698} colab_type="code" id="tGknYWlxo-9l" outputId="b25a5ff3-7239-45db-f3f8-160d51dcb38b"
fig = plt.figure(figsize=(16, 11))

plt.scatter(X[:, 0], X[:, 1])
arrowprops = dict(arrowstyle='->',
                  linewidth=2, shrinkA=0, shrinkB=0)
for length, vector in zip(pca.explained_variance_, pca.components_):
    print(vector)
    v = vector * 1 * np.sqrt(length)
    ax = plt.gca()
    ax.annotate('', pca.mean_ + v, pca.mean_, arrowprops=arrowprops)

plt.axis('equal')
plt.xlim(0, 2)
plt.ylim(0, 2)
plt.xlabel('x1')
plt.ylabel('x2')

# fig.savefig('ML_0515.png', bbox_inches='tight')
plt.show()
# plt.clf()

# + colab={"base_uri": "https://localhost:8080/", "height": 664} colab_type="code" id="M--XsTOZo-9n" outputId="f0029650-7b82-461a-e80e-14df679f30b4"
fig = plt.figure(figsize=(16, 11))

X_pca = pca.transform(X)
plt.scatter(X_pca[:, 0], X_pca[:, 1])

plt.annotate('', [0, 2], [0, 0], arrowprops=arrowprops)
plt.annotate('', [2, 0], [0, 0], arrowprops=arrowprops)

plt.axis('equal')
plt.xlim(-3, 3)
plt.ylim(-3, 3)
plt.xlabel('pca component 1')
plt.ylabel('pca component 2')

# fig.savefig('ML_0516.png', bbox_inches='tight')
plt.show()
# plt.clf()

# + colab={"base_uri": "https://localhost:8080/", "height": 497} colab_type="code" id="dxr12DIzo-9p" outputId="7d30abee-f9c7-4729-9d8e-26924551b929"
from sklearn.datasets import load_iris
from sklearn.decomposition import PCA
import matplotlib.pyplot as plt
import numpy as np

n_classes = 3
colors = 'byr'
CMAP = colors

iris = load_iris()

X = iris.data
y = iris.target

Y_target_names = iris.target_names


pca = PCA(n_components=2, whiten=True)
pca.fit(X)

#_________________________________________________________
fig = plt.figure(figsize=(12, 8))

X_pca = pca.transform(X)

for i, color in zip(range(n_classes), colors):
    idx = np.where(y == i)
    plt.scatter(X_pca[idx, 0], X_pca[idx, 1], label = Y_target_names[i],  c=color,
                edgecolor='black', lw=2, cmap=CMAP)

plt.axis("tight")
plt.xlabel('pca component 1')
plt.ylabel('pca component 2')
plt.legend(loc='upper center')

# fig.savefig('ML_0519.png', bbox_inches='tight')
plt.show()
# plt.clf()

# + [markdown] colab_type="text" id="5X0moN-Co-9r"
# ## Featureselektion

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" id="vtXJy9Hgo-9r" outputId="a9659352-9fb5-4088-e079-7646c5b66093"
from sklearn.datasets import load_iris
from sklearn.svm import SVC
import numpy as np

iris = load_iris()
X = iris.data
y = iris.target

# reference score
svc = SVC(kernel='linear', C=1)
clf = svc.fit(X, y)
print(clf.score(X, y))

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" id="O0lPqoFGo-9t" outputId="02f43923-2db2-403b-9059-46a305ad5de7"
# Add random noise as non informative data
rns = np.random.RandomState(12)
noise = rns.uniform(0, 6, size=(len(X), 1))
X = np.hstack([X, noise])

# Score with all noise
clf = svc.fit(X, y)
print(clf.score(X, y))

# + colab={} colab_type="code" id="EGnTye5ro-9v"
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import f_classif

selector = SelectKBest(f_classif, k=4)
X_sel = selector.fit_transform(X, y)

# + colab={"base_uri": "https://localhost:8080/", "height": 51} colab_type="code" id="BZXW3xZEo-9x" outputId="3af3d2d8-393b-4466-bfb2-6115ffb136ed"
print(selector.scores_)

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" id="X9icJTY6o-90" outputId="9aa9cf77-68fd-43fa-a689-6666ee833e10"
svc = SVC(kernel='linear', C=1)
clf = svc.fit(X_sel, y)
print(clf.score(X_sel, y))

# + [markdown] colab_type="text" id="Iy6elm8Qo-91"
# ## Selektion nach Tree-Modellen

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" id="KVrKK5pNo-92" outputId="a000c871-0d82-42cc-a7c9-e65c24c2bbe8"
from sklearn.feature_selection import SelectFromModel
from sklearn.ensemble import RandomForestClassifier

clf = RandomForestClassifier()
clf.fit(X, y)
print(clf.feature_importances_)

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" id="MNS96I_Ro-93" outputId="d6640305-5a6b-487f-c6f7-48e3ee2ff1bc"
selector = SelectFromModel(clf, threshold=0.02)
X_sel = selector.fit_transform(X, y)
print(selector.get_support())

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" id="masCa8pso-95" outputId="12ee5033-fbce-4733-f296-fa6fdd85a0fb"
svc = SVC(kernel='linear')
clf = svc.fit(X_sel, y)
print(clf.score(X_sel, y))

# + [markdown] colab_type="text" id="htdK7Awto-97"
# ## Rekursive Eliminierung nach Modellen

# + colab={"base_uri": "https://localhost:8080/", "height": 153} colab_type="code" id="06QeC-0Yo-97" outputId="d578ac96-4e96-4548-dc69-1b320b159991"
from sklearn.ensemble import RandomForestClassifier
clf = RandomForestClassifier(random_state=12)
clf.fit(X, y)

# + colab={} colab_type="code" id="wf-DKLjZo--A"
from sklearn.feature_selection import RFE
# Original selector = RFE(clf, 4)
selector = RFE(clf, n_features_to_select=4)

# + colab={} colab_type="code" id="w7VvfvFdo--D"
selector = selector.fit(X, y)

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" id="GDjg3CvSo--F" outputId="ab75bb52-e9e3-4887-803f-711d829262a1"
print(selector.get_support())

StarcoderdataJupyterScriptsDedupFiltered