# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
#Libraries used for this mini-project
import pandas as pd
from pandas.plotting import scatter_matrix

import numpy as np

import matplotlib.pyplot as plt
from matplotlib.gridspec import GridSpec

import sklearn.datasets
from sklearn.model_selection import StratifiedShuffleSplit, GridSearchCV
from sklearn.preprocessing import OneHotEncoder, StandardScaler, OrdinalEncoder
from sklearn.pipeline import Pipeline
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import confusion_matrix
import sklearn.metrics

import torch
import torch.nn.functional as F
# -

# # Loading Data

iris = sklearn.datasets.load_iris()
iris.keys()

# +
data = iris["data"]
target = iris["target"]
features = [col.split(" (")[0].replace(" ", "_") for col in iris["feature_names"]]
target_names = iris["target_names"]
target = [target_names[i] for i in target]#Getting class of the iris flower


plt.hist(target)
plt.xticks(target_names)
plt.show()
# -

data_df = pd.DataFrame(data, columns=features)
data_df.describe()

# # Data Transformation

# +
#Checking if we have missing values
for col in data_df.columns:
    if data_df[col].isna().values.sum() > 0:
        print("Missing data in column" + col)
    else:
        print("No missing data at " + col)

flag = sum([v == np.nan for v in np.asarray(target)])
if flag:
    print("Missing values in target")
else:
    print("No missing values in target")
    
# -

scatter_matrix(data_df, figsize=(10, 10))# It seems that there is linear dependency between petal length and petal width
plt.show()

corr = data_df.corr()#see the correlation between petal length, petal width, sepal length
corr

#Lets see the distribution of the predictors within flower classes
ax = pd.plotting.boxplot(data=data_df, by=target, figsize=(10, 10))
for x in ax:
    x[0].set_xlabel("")
plt.suptitle("")
plt.xlabel("")
plt.show()

# # Relationship Between Predictors

# +
#Testing Seperability by using linear hyperplane
gs = GridSpec(3, 2)
axes = []
combination = {}

        
for i in range(0, len(features) - 1):
    for j in range(i + 1, len(features)):
        if (features[i], features[j]) not in combination:
            combination[(features[i], features[j])] = 0
        if (features[j], features[i]) not in combination:
            combination[(features[j], features[i])] = 1
#dict comprehension
combination = {k: v for k,v in combination.items() if v ==0}

#Getting colors
color_dict = {"setosa": "red", "versicolor": "blue", "virginica": "green"}
colors = [color_dict[i] for i in target]
#initializing the axes and plotting scatter plot
counter = 0
keys = list(combination.keys())
for row in range(0, 3):
    for col in range(0, 2):
        axes.append(plt.subplot(gs[row, col]))
        x_d, y_d = keys[counter]
        data_df.plot(kind="scatter", x=x_d, y=y_d, ax=axes[row * 2 + col], c=colors)
        axes[row * 2 + col].set_yticks([])
        axes[row * 2 + col].set_xticks([])
        counter += 1
#As expected petal_lengthxpetal_width shows good seperation and similar both works well with sepal length
# -

# # Train Test Split

features_full = [col for col in data_df.columns]
features_full.append("class")
full_data = pd.DataFrame(np.c_[data_df, target], columns=features_full)

strata = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)
for train_ind, test_ind in strata.split(full_data[features], full_data["class"]):
    train_data = full_data.iloc[train_ind]
    test_data = full_data.iloc[test_ind]

print(train_data.shape)
print(test_data.shape)

# # Standard Normalize Data and building Model

# +
X_train = train_data[features].to_numpy()
y_train = train_data["class"].to_numpy().reshape(-1, 1)

num_pipeline = Pipeline([
    ("standarize_data", StandardScaler())
])
cat_pipeline = Pipeline([
    ("ordinal encoder", OrdinalEncoder())
])

X_train_tr = num_pipeline.fit_transform(X_train)
y_train_tr = cat_pipeline.fit_transform(y_train)

# -

model = LogisticRegression(multi_class="multinomial", C=0.9)
model.fit(X_train_tr, y_train_tr)

preds = model.predict_proba(X_train_tr).argmax(axis=1)
confusion_matrix(preds, y_train_tr)

# +
#model performance on test set
X_test = test_data[features].to_numpy()
y_test = test_data["class"].to_numpy().reshape(-1, 1)

X_test_tr = num_pipeline.transform(X_test)
y_test_tr = cat_pipeline.transform(y_test)

preds = model.predict_proba(X_test_tr).argmax(axis=1)
confusion_matrix(preds, y_test_tr)# So, it is working well with regularizing factor 1/0.9
# -

# # Checking if reduced model perform as well as the complete model

# +
#Lets if we reduced the model by removing the sepal width
X_train = train_data[["petal_length", "petal_width", "sepal_length"]].to_numpy()
X_test = test_data[["petal_length", "petal_width", "sepal_length" ]].to_numpy()


X_train_tr = num_pipeline.fit_transform(X_train)
X_test_tr = num_pipeline.transform(X_test)


model = LogisticRegression(multi_class="multinomial", C=0.9)
model.fit(X_train_tr, y_train_tr);
preds = model.predict_proba(X_train_tr).argmax(axis=1)
confusion_matrix(preds, y_train_tr)
# -

preds = model.predict_proba(X_test_tr).argmax(axis=1)
confusion_matrix(preds, y_test_tr)# So, it is working well with regularizing factor 1/0.9

# # Finding Better regularizing value

# +
X_train = train_data[features].to_numpy()
X_test = test_data[features].to_numpy()

X_train_tr = num_pipeline.fit_transform(X_train)
X_test_tr = num_pipeline.transform(X_test)


model = LogisticRegression()
params_grid = [
    {"C": np.linspace(0.1, 1.0, 10)}
]
score = sklearn.metrics.make_scorer(sklearn.metrics.precision_score, average = 'weighted')#Use precision score for the three classes
grid_search = GridSearchCV(estimator=model, param_grid=params_grid, scoring=score, cv=5)
grid_search.fit(X_train_tr, y_train_tr.ravel())
# -

est = grid_search.best_estimator_

preds = est.predict_proba(X_test_tr).argmax(axis=1)
confusion_matrix(preds, y_test_tr)

for score, param in zip(grid_search.cv_results_["mean_test_score"], grid_search.cv_results_["params"]):
    print("Score: ", score, end="\t")
    print(param)

grid_search.best_estimator_

grid_search.cv_results_

# # Using PyTorch 

# +
features = [col for col in train_data.columns if col != "class"]
X_train = np.asarray(train_data[features], dtype=np.float32)
y_train = np.asarray(train_data["class"]).reshape(-1, 1)
X_test = np.asarray(test_data[features], dtype=np.float32)
y_test = np.asarray(test_data["class"]).reshape(-1, 1)

ordinal = OrdinalEncoder()
ordinal.fit(y_train)
y_train = ordinal.transform(y_train)
y_test = ordinal.transform(y_test)
print(y_test[0:5])
ordinal.categories_
# -

X_train_tensor = torch.from_numpy(X_train)
X_test_tensor = torch.from_numpy(X_test)
y_train_tensor = torch.from_numpy(y_train)
y_test_tensor = torch.from_numpy(y_test)


class IrisClassifier(torch.nn.Module):
    
    def __init__(self, num_features, *args, **kwargs):
        super(IrisClassifier, self).__init__()
        self.l1 = torch.nn.Linear(num_features, 50)
        self.l2 = torch.nn.Linear(50, 50)
        self.l3 = torch.nn.Linear(50, 20)
        self.l4 = torch.nn.Linear(20, 3)
    
    def forward(self, x):
        x = F.relu(self.l1(x))
        x = F.relu(self.l2(x))
        x = F.relu(self.l3(x))
        x = F.dropout(x, p=0.4)
        x = self.l4(x)
        x = F.softmax(x, dim=1)
        x = x.view(x.shape[0], 3)
        #x = x.float()
        #x = torch.argmax(x, dim=1)
        #print(x.shape)
        #val, ind = torch.max(x, dim=1)
        #print(x)
        return x


# +
model = IrisClassifier(len(features))
#initializing model
def init_model(layer):
    if type(layer) == "Linear":
        torch.nn.init.xavier_normal(layer.weight)
        layer.bias.data.fill_(0.001)

loss_fn = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)

training_error = []
for epoch in range(0, 1000):
    y_pred = model(X_train_tensor)
    #print(y_pred)
    loss = loss_fn(y_pred, torch.squeeze(y_train_tensor).type(torch.LongTensor))
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()
    training_error.append(loss.item())
    if epoch%10 == 0:
        print(f"epoch {epoch} the log-likelihood is {loss.item()}")
# -

for param in model.parameters():
    print(param)

y_p = torch.argmax(y_pred, dim=1)

y_p

confusion_matrix(y_p, y_train_tensor)

# # Prediction for test data

y_test_pred = model(X_test_tensor)
y_test_p = torch.argmax(y_test_pred, dim=1)
confusion_matrix(y_test_p, y_test_tensor)



StarcoderdataJupyterScriptsDedupFiltered