# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] colab_type="text" id="37puETfgRzzg"
# # Data Preprocessing Tools

# + [markdown] colab_type="text" id="EoRP98MpR-qj"
# ## Importing the libraries

# + colab={} colab_type="code" id="N-qiINBQSK2g"
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

# + [markdown] colab_type="text" id="RopL7tUZSQkT"
# ## Importing the dataset

# + colab={} colab_type="code" id="WwEPNDWySTKm"
dataset = pd.read_csv('Data.csv') # pandas dataframe - each column is a categorical variable

# Every machine learning model has features and labels.
# Features(independent variables) are inputs, label(dependent variables/target) are output

# last column --> labels, all columns except last column --> features

# X --> features, y --> label

# iloc[r:r,c:c]

# [:] --> no range, means take everything 
# [:, :-1] --> take all rows, all columns excluding last one (uppper bound not included in slicing)
# [:, -1] --> take all rows, of last column 
# -1 --> index of last column 

# Create maxtrix of features and label variables 

X = dataset.iloc[:, :-1].values
y = dataset.iloc[:, -1].values

# + colab={"base_uri": "https://localhost:8080/", "height": 188} colab_type="code" id="hCsz2yCebe1R" outputId="1e4cc568-4e51-4b38-9d46-4aa3f15204be"
print(X)

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" id="eYrOQ43XcJR3" outputId="e0873b2a-3b08-4bab-ef0d-15b88858ca44"
print(y)

# + [markdown] colab_type="text" id="nhfKXNxlSabC"
# ## Taking care of missing data

# +
# Checking for missing data

np.isnan(np.sum(X[:, 1:3]))

# + colab={} colab_type="code" id="c93k7ipkSexq"
from sklearn.impute import SimpleImputer
imputer = SimpleImputer(missing_values=np.nan, strategy='mean')


# select only the numberical columns 

imputer.fit(X[:, 1:3]) # fit() will look at NaN values, and will compute mean
X[:, 1:3] = imputer.transform(X[:, 1:3]) # transform() will replace NaN by mean(replace)

# update columns with returned by transform()
# -

np.isnan(np.sum(X[:, 1:3]))

# + colab={"base_uri": "https://localhost:8080/", "height": 188} colab_type="code" id="3UgLdMS_bjq_" outputId="254af4e0-681e-47f5-aaa7-b9c6f43258e9"
print(X)

# + [markdown] colab_type="text" id="CriG6VzVSjcK"
# ## Encoding categorical data

# + [markdown] colab_type="text" id="AhSpdQWeSsFh"
# ### Encoding the Independent Variable

# + colab={} colab_type="code" id="5hwuVddlSwVi"
# Ml models cannot learn from strings. It can only learn from binary numbers - 0 and 1
# so we will convert all columns of str to int 

# note: some columns have type object. Conversion --> object to str to int

# convert categorical variables to dummy variables (0 or 1)

from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import OneHotEncoder

# [0,1,2] --> indexes of the columns
ct = ColumnTransformer(transformers=[('encoder', OneHotEncoder(), [0])], remainder='passthrough')
X = np.array(ct.fit_transform(X)) # fit() and transform() at the same time 

#  remainder='passthrough' --> columns, other than the ones specified, will remain untouched

# + colab={"base_uri": "https://localhost:8080/", "height": 188} colab_type="code" id="f7QspewyeBfx" outputId="5b35feef-7fe2-46ef-ce70-80495f94f4ed"
print(X)

# + [markdown] colab_type="text" id="DXh8oVSITIc6"
# ### Encoding the Dependent Variable

# + colab={} colab_type="code" id="XgHCShVyTOYY"
# Replace "Yes" or "No" by binary variables

# Label Encoder is only for labels/output
from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()
y = le.fit_transform(y) # send one single matrix/vector, y doesn't necessarily need to be a numpy array

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" id="FyhY8-gPpFCa" outputId="7f76ef29-5423-4c3e-cf69-45fbc366a997"
print(y)

# + [markdown] colab_type="text" id="qb_vcgm3qZKW"
# ## Splitting the dataset into the Training set and Test set

# + colab={} colab_type="code" id="pXgA6CzlqbCl"
from sklearn.model_selection import train_test_split

# test size = 0.2 --> Train = 80% of data, test = 20% of data
# random_state = 0 --> to keep the random split the same - to make results reproducable -- like np.random.seed()

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 1)

# Train set --> train model on existing observations
# Test set --> evaluate model on new observations (future data)

# X_train corresponds to y_train
# X_test corresponds to y_test
# -

X_train.shape, y_train.shape, X_test.shape, y_test.shape

# + colab={"base_uri": "https://localhost:8080/", "height": 154} colab_type="code" id="GuwQhFdKrYTM" outputId="de1e527f-c229-4daf-e7c5-ea9d2485148d"
print(X_train)

# + colab={"base_uri": "https://localhost:8080/", "height": 51} colab_type="code" id="TUrX_Tvcrbi4" outputId="9a041a9b-2642-4828-fa2f-a431d7d77631"
print(X_test)

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" id="pSMHiIsWreQY" outputId="5afe91e0-9244-4bf5-ec1b-e3e092b85c08"
print(y_train)

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" id="I_tW7H56rgtW" outputId="2a93f141-2a99-4a69-eec5-c82a3bb8d36b"
print(y_test)

# + [markdown] colab_type="text" id="TpGqbS4TqkIR"
# ## Feature Scaling

# + colab={} colab_type="code" id="AxjSUXFQqo-3"
# Scale features to make sure all values are in the same scale (Prevent one feature from dominating the other)

# Feature scaling is done after train,test split (to prevent information leakage)

# Scaling on both train and test set must be same (Better to do at once). Although they are done seperately
# Scaling is not done on y(labels)

# Feature scaling is not needed for all ML algorithms. Only for some.

from sklearn.preprocessing import StandardScaler
sc = StandardScaler()
X_train[:, 3:] = sc.fit_transform(X_train[:, 3:]) # Take all rows, and columns from 3rd to last (excluding dummy variable columns) 
X_test[:, 3:] = sc.transform(X_test[:, 3:]) # only apply transform() because test state must be applied the same scale as train state
# in order to get the same transformation. (fit_transform would have given a new scale)

# IMP: Standardisation is not done on dummy variables (variables that have been one hot encoded)
# because we lose our numerical values

# Only apply them on numerical variables 


# + colab={"base_uri": "https://localhost:8080/", "height": 154} colab_type="code" id="DWPET8ZdlMnu" outputId="dea86927-5124-4e2a-e974-2804df9a913c"
print(X_train)

# + colab={"base_uri": "https://localhost:8080/", "height": 51} colab_type="code" id="sTXykB_QlRjE" outputId="b68f0cfc-d07c-48cb-80d0-6800028c41f9"
print(X_test)
# -



StarcoderdataJupyterScriptsDedupFiltered