# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import pandas as pd 
import numpy as np 
import seaborn as sns 
import matplotlib.pyplot as plt
# %matplotlib inline

df=pd.read_csv('final_test.csv')
df

df.info()

df.describe()

df.shape

df.head()

df.isnull()

df.isnull().any()

df.isnull().sum()

# +
#I have noticed that we have some missing data regarding the following points (age and height)>> both numerical data
#Therefore we have used .any(), now we need to use dropna to fill the missing ones.
# -

#To handle this we have many ways (6)
#1. delete the rows wih missing valuse by using (Not valid if we have a small dataset)
dataframe.dropna(inplace=True)
#https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.dropna.html?highlight=dropna

df.dropna(inplace=True)

df.isnull()

df.isnull().any()
#As we see there are no more missing valuse in the data set (all false)

df.isnull().sum()

# +
#OR To fill the missing with new data 
#Apply classifier algorithm 
#1.Mean/ Median /Mode imputation
#2.Random Sample Imputation
#3.Capturing NAN values with a new feature with (0 or 1)
#4.End of Distribution imputation
#5.KNN Imputer
#https://anurodhchoudhary.medium.com/how-to-handle-numerical-missing-values-in-data-with-implementation-7d034884795e#:~:text=Missing%20Data%20can%20occur%20when%20no%20information%20is,to%20as%20NA%20%28Not%20Available%29%20values%20in%20pandas.

# +
#If there is suspicion that the missing value is not at random then capturing that information is important
#therefore we have to make disturbuition for the data before deciding 
# -

df.shape

df.age.hist(bins=50)

df.height.hist(bins=50)

df.weight.hist(bins=50)

# +
#Next step is to work with nun numerical values (size)
#This would be by using LabelEncoder
# -

from sklearn.preprocessing import LabelEncoder

# label_encoder object
label_encoder =LabelEncoder()
# Encode labels in column. 
df['size']= label_encoder.fit_transform(df['size'])
df.head(10)

df.tail()

# +
#Next step is to detect and handle outliers 
# -

sns.pairplot(df)

sns.distplot(df['size'],color='m', kde=False, bins=30)

df.size

df["size"].value_counts()

#The results before the coversion.
M       29712
S       21924
XXXL    21359
XL      19119
L       17587
XXS      9964
XXL        69

sns.distplot(df['height'],color='r', kde=False, bins=30)

sns.distplot(df['weight'],color='g', kde=False, bins=30)

sns.distplot(df['age'],color='b', kde=False, bins=30)

#We have to use Zscore and detect the 4 quartiel, in order to get rid of the outliers
#1st we use boxplot 
sns.boxplot(df['height'])

sns.boxplot(df['weight'])

sns.boxplot(df['age'])

#Z score = (x -mean) / std. deviation
x=df[['weight','age','height']]
x

y=df[['size']]
y

#data = [1, 2, 2, 2, 3, 1, 1, 15, 2, 2, 2, 3, 1, 1, 2]
#mean = np.mean(data)
#std = np.std(data)
#print('mean of the dataset is', mean)
#print('std. deviation is', std)
mean=np.mean(df)
std=np.std(df)
print('mean of the data is',mean)
print('std of the data is',std)

df.describe()

#Z score = (x -mean) / std. deviation
#Hier detecting outliers by std only 
upper_limit_h=df.height.mean() + 3*df.height.std()
print('upper_limit_h is',upper_limit_h)
lower_limit_h=df.height.mean() - 3*df.height.std()
print('lower_limit_h is',lower_limit_h)

upper_limit_w=df.weight.mean() + 3*df.weight.std()
print('upper_limit_w',upper_limit_w)
lower_limit_w=df.weight.mean() - 3*df.weight.std()
print('lower_limit_w',lower_limit_w)

upper_limit_a=df.age.mean() + 3*df.age.std()
print('upper_limit_a is', upper_limit_a)
lower_limit_a=df.age.mean() - 3*df.age.std()
print('lower_limit_a is', lower_limit_a)

df[(df.age>upper_limit_a) | (df.age<lower_limit_a)]

df['zscore']=(df.height-df.height.mean())/df.height.std()
df.head(10)

df['zscore']


#Now let's remove the outliers 
#https://www.youtube.com/watch?v=Vc4cXIAa69Y
def outliers(df, ft):
    Q1=df[ft].quantile(0.25)
    Q3=df[ft].quantile(0.75)
    IQR= Q3 - Q1
    
    lower_bound= Q1 - 1.5 * IQR
    upper_bound= Q3 + 1.5 * IQR
    
    st = df.index[ (df[ft] < lower_bound) | (df[ft] > upper_bound)]
    
    return st


index_list=[]
for i in ['height', 'weight', 'age']:
    index_list.extend(outliers(df,i))

#This is list is the outliers 
index_list

print("DataFrame shape before delete outliers is. ", df.shape)
for outliers in index_list:
    df.drop(outliers, inplace=True)
print("DataFrame shape after deleting outliers. ",df.shape)

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)

# +
scaler= StandardScaler()

scaler.fit(x_train)

x_train = scaler.transform(x_train)
x_test = scaler.transform(x_test)
# -

from sklearn.datasets import load_iris

#from sklearn.metrics import classification_report
#from sklearn.metrics import confusion_matrix
#from sklearn.metrics import accuracy_score
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVR
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor

models = {
    "KNNR" : KNeighborsClassifier(), 
    "SVR": SVR(),
    "DT": DecisionTreeRegressor(),
    "RF": RandomForestRegressor(),
}

for name, model in models.items():
    print(f'Using model: {name}')
    model.fit(x_train, y_train)
    print(f'Training Score: {model.score(x_train, y_train)}')
    print(f'Test Score: {model.score(x_test, y_test)}')  
    print('-'*30)

# +
#As we can see the oultiers affect badly on the results!
#But best result goes to RF
model = RandomForestRegressor()

model.fit(x_train, y_train)
# -

print(x_test)

y_pred = model.predict(x_test)
y_pred

y_test

x.columns

# +
#Trying to use a calssifier 
#https://www.activestate.com/resources/quick-reads/how-to-classify-data-in-python/
#Since we have more than 2 values, we need to use Multi-calss classification such as:
#Decision Tree/Random Forest, K-Nearest Neighbors (KNN), Linear Discriminant Analysis, Support Vector Machines (SVMs)
#Since we have already used DT, KNN and SVM previously! so we can try 
# -

# Import libraries and classes required for this example:
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler 
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import classification_report, confusion_matrix

# +
# Standardize features by removing mean and scaling to unit variance:
scaler = StandardScaler()
scaler.fit(x_train)

x_train = scaler.transform(x_train)
x_test = scaler.transform(x_test)

# +
# Use the KNN classifier to fit data:
classifier = KNeighborsClassifier(n_neighbors=5)
classifier.fit(x_train, y_train) 

# Predict y data with classifier: 
y_predict = classifier.predict(x_test)

# Print results: 
print(confusion_matrix(y_test, y_predict))
print(classification_report(y_test, y_predict))

# +
from sklearn.metrics import r2_score

print("R2_Score is: ", r2_score(y_test, y_pred).round(4))

# +
#The result is very poor which means the model will be very bad in predicting 
# -



StarcoderdataJupyterScriptsDedupFiltered