# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: 'Python 3.6.6 64-bit (''base'': conda)'
#     name: python36664bitbaseconda80ab881b08fc4123973af10a07e0b9cc
# ---

# import library
import matplotlib.pyplot as plt
import numpy as np
import scipy.optimize as opt
import sklearn.linear_model
import sklearn.model_selection
import pandas as pd
import seaborn as sns

# ### In the assignment you should:
#
# - Load the data and check its correctness
# - Explore the basic parameters: how many data points do we have? What are the targets and what is their distribution? Any kind of exploratory data analysis is welcome
# - Identify the problem: is it regression? classification?
# - Identify metric you're going to use
# - Design and run the experiment: train and validate your model
# - Compare your results with some kind of baseline (simplest possible solution to the problem)
# - (Optional) estimate feature importances and select the most important features
#
#
#

# ### load data and check its correctness

# + tags=[]
# 
pd_data = pd.read_csv("FINAL-data/winequality-red.csv")
nRow, nCol = pd_data.shape
print(f'Wine quality : There are {nRow} rows and {nCol} columns')
# -

# check column types
pd_data.dtypes

# ### Explore the basic parameters: how many data points do we have? What are the targets and what is their distribution? 

# get the data description
pd_data.describe()

# check missing data
pd_data.isna().sum()

pd_data.corr()

# +
#PEARSON CORRELATION

plt.figure(figsize = (15,10))
sns.heatmap(pd_data.corr(method="pearson"))
plt.title('PEARSON CORRELATION', fontsize=15)
# -

fig, axes = plt.subplots(nrows=6,ncols=2)
fig.set_size_inches(12, 10)
sns.distplot(pd_data["fixed acidity"],ax=axes[0][0])
sns.distplot(pd_data["volatile acidity"],ax=axes[0][1])
sns.distplot(pd_data["citric acid"],ax=axes[1][0])
sns.distplot(pd_data["residual sugar"],ax=axes[1][1])
sns.distplot(pd_data["chlorides"],ax=axes[2][0])
sns.distplot(pd_data["free sulfur dioxide"],ax=axes[2][1])
sns.distplot(pd_data["total sulfur dioxide"],ax=axes[3][0])
sns.distplot(pd_data["density"],ax=axes[3][1])
sns.distplot(pd_data["pH"],ax=axes[4][0])
sns.distplot(pd_data["sulphates"],ax=axes[4][1])
sns.distplot(pd_data["alcohol"],ax=axes[5][0])
sns.distplot(pd_data["quality"],ax=axes[5][1])

# ### Identify the problem: is it regression? classification?
# It is a classification problem. Quality of wine can be divided into several class with only discrete numbers.

# + tags=[]

# remove columns with low correlation.
pd_data = pd_data.drop(["residual sugar", "free sulfur dioxide", "pH"], axis=1)


#log some columns to let them have a normal distribution
pd_data["log_total_SO2"] = np.log1p(pd_data["total sulfur dioxide"])
pd_data["log_alcohol"] = np.log1p(pd_data["alcohol"])

pd_data = pd_data.drop(["total sulfur dioxide", "alcohol", "fixed acidity"], axis=1)
pd_data.corr()

# + tags=[]
X = pd_data.drop("quality", axis=1)
Y = pd_data["quality"]

print(X.shape)
print(Y.shape)
# -

from imblearn.over_sampling import SMOTE
sm=SMOTE()
X,Y=sm.fit_sample(X,Y)

# ### Identify metric you're going to use
# Accuracy of prediction will be the only metric of measuring classification problem.

from sklearn.preprocessing import StandardScaler, MinMaxScaler
scaler = StandardScaler().fit(X)
scaled_X = scaler.transform(X)

# + tags=[]
from sklearn.model_selection import train_test_split

seed      = 42
test_size = 0.20

X_train, X_test, Y_train, Y_test = train_test_split(scaled_X, Y, test_size = test_size, random_state = seed)

print(X_train.shape)
print(X_test.shape)
print(Y_train.shape)
print(Y_test.shape)
# -

# ### Design and run the experiment: train and validate your model

# + tags=[]
from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score
from matplotlib.colors import ListedColormap
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.datasets import make_moons, make_circles, make_classification
from sklearn.neural_network import MLPClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
from xgboost import XGBClassifier
from lightgbm import LGBMClassifier

import time
import datetime

start = 0
end = 0
start = time.time()

# user variables to tune
folds   = 10

# hold different regression models in a single dictionary
models = {}
models["DecisionTree"]  = DecisionTreeClassifier()
models["KNN"]           = KNeighborsClassifier()
models["RandomForest"]  = RandomForestClassifier()
models["AdaBoost"]      = AdaBoostClassifier()
models["GradientBoost"] = GradientBoostingClassifier()
models["XGBoost"] = XGBClassifier()
models["LGBM"] = LGBMClassifier()
models["GaussianNB"] = GaussianNB()
models["Quad"] = QuadraticDiscriminantAnalysis()


# 10-fold cross validation for each model
model_results = []
model_names   = []
for model_name in models:
	model   = models[model_name]
	k_fold  = KFold(n_splits=folds, random_state=seed)
	results = cross_val_score(model, X_train, Y_train, cv=k_fold)
	
	model_results.append(results)
	model_names.append(model_name)
	print("{}: {}, {}".format(model_name, round(results.mean(), 3), round(results.std(), 3)))
	end = time.time()
	list_lapse = end - start
	print("Time taken for processing {}: {}".format(model_name, str(datetime.timedelta(seconds=list_lapse))))

# box-whisker plot to compare regression models
figure = plt.figure(figsize = (20,8))

figure.suptitle('Classification models comparison')
axis = figure.add_subplot(111)
plt.boxplot(model_results)
axis.set_xticklabels(model_names, rotation = 45, ha="right")
axis.set_ylabel("Accuracy Score")
plt.margins(0.05, 0.1)

# + tags=[]
# from sklearn.model_selection import RandomizedSearchCV
# model = RandomForestClassifier(random_state = 42)
# random_grid = {'bootstrap': [True, False],
#                'max_depth': [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, None],
#                'max_features': ['auto', 'sqrt'],
#                'min_samples_leaf': [1, 2, 4],
#                'min_samples_split': [2, 5, 10],
#                'n_estimators': [130, 180, 230]}

# print('Parameters currently in use:\n')
# print(model.get_params())         


# #print random grid

# print(random_grid)

# rf_random = RandomizedSearchCV(estimator = model, param_distributions = random_grid, n_iter = 100, cv = 5, verbose=2, random_state=42, n_jobs = -1)
# # Fit the random search model
# rf_random.fit(X_train, Y_train)

# rf_random.best_params_
# -

# ### Compare your results with some kind of baseline (simplest possible solution to the problem)

# + tags=[]
from sklearn.metrics import accuracy_score
import math
import statistics

#Predicting TEST 
model = RandomForestClassifier(random_state = 42)
model.fit(X_train, Y_train)
test_predict = model.predict(X_test)

accuracy = accuracy_score(Y_test, test_predict)
print("Prediction accuracy of RandomForest. Mean: {}, SD: {}".format(round(accuracy.mean(), 3), round(accuracy.std(), 3)))

avg_quality = round(statistics.mean(Y_test))
avg_quality_list = [avg_quality for i in range(Y_test.size)]
base_accuracy = accuracy_score(Y_test, avg_quality_list)
print("Simple prediction by taking average of all quality score. Mean: {}, SD {}".format(round(base_accuracy.mean(), 3), round(base_accuracy.std(), 3)))

# + tags=[]
from sklearn.metrics import accuracy_score,classification_report,confusion_matrix
print(accuracy_score(Y_test,test_predict))
print(classification_report(Y_test,test_predict))
sns.heatmap(confusion_matrix(Y_test,test_predict),annot=True,cmap="Purples")

# + tags=[]
import sklearn.metrics 
real = [9,7,5,4]
prediction = [10,8,4,9]

#1,1,1,5 = 8/4 =2

print(sklearn.metrics.mean_absolute_error(real,prediction))
print(sklearn.metrics.mean_relative_error(real,prediction))


# -



StarcoderdataJupyterScriptsDedupFiltered