# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="i621pct-sQRS" colab_type="text"
# #`This notebook is an Illustration to create a Support Vector Machine with Sklearn using Python 🐍`
#
# For more information, check out the [SVM](https://scikit-learn.org/stable/modules/generated/sklearn.svm.SVC.html).

# + [markdown] id="ChVjcR7A5XhQ" colab_type="text"
# ## **Setup**
# Let's setup the imports:   

# + id="bq-sR6U9WFkY" colab_type="code" colab={}
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
# %matplotlib inline

import seaborn as sns
from matplotlib.colors import ListedColormap

from sklearn.svm import SVC
from sklearn.svm import LinearSVC
from sklearn.svm import SVR
from sklearn.model_selection import GridSearchCV

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler

from sklearn.metrics import accuracy_score, classification_report, confusion_matrix 


# custom color maps
cm_dark = ListedColormap(['#ff6060', '#8282ff','#ffaa00','#fff244','#4df9b9','#76e8fc','#3ad628'])
cm_bright = ListedColormap(['#ffafaf', '#c6c6ff','#ffaa00','#ffe2a8','#bfffe7','#c9f7ff','#9eff93'])  

# + [markdown] id="zX9NXw9F5Y8D" colab_type="text"
# # **Datatset**
# Let's load the Mobile Price Classification dataset which is uploaded on my Google Drive.

# + id="CvXzu177WaD2" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 151} outputId="75dec3b2-5a65-462d-c810-1d4fa19f6477"
# !gdown --id 17wpxB4XilNQ54Yf7aYrpK6vTrvE7f5Gh 
# !gdown --id 1hxdj2JGckweySFIJluiCztjAndsNLfGG 

# + [markdown] id="CIAQkNVm5ttK" colab_type="text"
# Let's read the CVS files: 

# + id="PdyMWFhOWhfb" colab_type="code" colab={}
df = pd.read_csv("/content/train.csv")
test = pd.read_csv("/content/test.csv") 

# + [markdown] id="zS9nVrbn5wtr" colab_type="text"
# First few rows in the DataFrame:  

# + id="krJXVAUpY8f_" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 215} outputId="fd2e7c47-00d0-4441-8ff8-5154eef150ab"
df.head()

# + [markdown] id="cfhygrUu5zEo" colab_type="text"
# Let's see if we have any null values: 

# + id="SNgAFdZ6Ytp3" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="9c95f818-bc07-44b8-8e0f-667a052d260b"
df.shape

# + id="102gHGtlZES-" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 386} outputId="dd5fa28f-e42e-4adb-b415-573743b8a33c"
df.isnull().sum() 

# + [markdown] id="jlpRlyU0cQj7" colab_type="text"
# Now in the data set there is no need to create dummy variables or handle missing data as data set doesn't have any missing data.

# + [markdown] id="hbimcPCj52Kf" colab_type="text"
# Let's see the columns present in our dataFrame:

# + id="BPryNZDSZInV" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 101} outputId="47b48a87-b170-49f0-d4f1-b3af28df4ee4"
df.columns 

# + [markdown] id="mw0_-ToC6qA1" colab_type="text"
# Let's set our variables for training from our dataFrame into numpy arrays:

# + id="dTe2VtyqZKlX" colab_type="code" colab={}
y_t = np.array(df['price_range']) 
x_t = df
x_t = df.drop(['price_range'], axis=1)
x_t = np.array(x_t)  

# + id="oM55zJnzbWIF" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 50} outputId="da2f09e8-3f37-40b1-b3c4-5718eabeca2b"
print("shape of x :"+str(x_t.shape))
print("shape of y :"+str(y_t.shape))

# + [markdown] id="1szUD2vU64if" colab_type="text"
# Let's scale our numpy variables using sklearn's MinMaxScalar:
#
# Reference: [MinMaxScalar](https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.MinMaxScaler.html) 

# + id="i-ciSalNj8p6" colab_type="code" colab={}
scaler = MinMaxScaler() 
x_t = scaler.fit_transform(x_t)  

# + id="XYbkY6qdZvLE" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 84} outputId="a74947e4-8f47-4167-a5e8-b781a2450a16"
X_train,X_test,Y_train,Y_test = train_test_split(x_t, y_t, test_size=.20, random_state=42)
print("shape of X Train :"+str(X_train.shape))
print("shape of X Test :"+str(X_test.shape))
print("shape of Y Train :"+str(Y_train.shape))
print("shape of Y Test :"+str(Y_test.shape))

# + id="v1tHdraxddNg" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 50} outputId="c959d151-e07f-476c-f78f-1eb0bf2d52ca"
# plotting the decision boundries for the data 
#converting the data to array for plotting.
x = np.array(df.iloc[:,[0,13]])
y = np.array(df['price_range'])
print("Shape of x:"+str(x.shape))
print("Shape of y:"+str(y.shape))
x = scaler.fit_transform(x)

# + id="hq2XGJlmbVsZ" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 281} outputId="5b213224-556c-463e-d88c-b3408781fbfa"
plt.scatter(x[:,0], x[:,1], c = y, cmap = cm_dark, s=15, label = y) 
plt.title("Scatter Plot", fontweight='bold')
plt.show(); 

# + [markdown] id="wavOzFkC7P2X" colab_type="text"
# Let's split our variables into train and test sets:

# + [markdown] id="dPVR8sJl9KFb" colab_type="text"
# ## Support Vector Machine with Linear Kernel 

# + id="euJqUd-8Z38U" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 420} outputId="6aae0457-99da-42fd-e17c-8fddf3cf9690"
for c_values in [1, 2, 3, 5,10, 20, 40, 60, 80, 100, 150, 200]:
  clf = SVC(kernel='linear', C = c_values).fit(X_train, Y_train) 
  scoretrain = clf.score(X_train, Y_train)
  scoretest = clf.score(X_test, Y_test) 
  print("Linear SVC of C: {}, Training Score: {:2f}, Testing Score: {:2f}\n".format(c_values, scoretrain, scoretest)) 

# + id="P0Lv9hwyGioa" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 350} outputId="fe4e5197-67bf-4871-ae7e-08a1205ce85b"
y_pred = clf.predict(X_test)  
cm = confusion_matrix(y_pred, Y_test) 
fig, ax = plt.subplots(figsize=(8,5))
sns.heatmap(cm, annot = True, cmap = 'YlGnBu', fmt='g'); 
plt.title('Confusion matrix', fontweight='bold')
plt.ylabel('Actual label', fontweight='bold')
plt.xlabel('Predicted label', fontweight='bold'); 

# + id="3PJBX4M0cTwU" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 281} outputId="f97473bd-7376-4feb-80b5-5e43e907185e"
h =0.02
C_param = 1
clf1 = SVC(kernel = 'linear', C = C_param)
clf1.fit(x, y)

# Plot the decision boundary. For that, we will assign a color to each
# point in the mesh [x_min, x_max]x[y_min, y_max].
x_min, x_max = x[:, 0].min()-.20, x[:, 0].max()+.20
y_min, y_max = x[:, 1].min()-.20, x[:, 1].max()+.20
xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
Z = clf1.predict(np.c_[xx.ravel(), yy.ravel()])   # ravel to flatten the into 1D and c_ to concatenate 

# Put the result into a color plot
Z = Z.reshape(xx.shape)
plt.figure()
plt.pcolormesh(xx, yy, Z, cmap=cm_bright)

# Plot also the training points
plt.scatter(x[:, 0], x[:, 1], c=y, cmap=cm_dark, edgecolor='k', s=20)
plt.xlim(xx.min(), xx.max())
plt.ylim(yy.min(), yy.max())
plt.title("SVM Linear Classification (kernal = linear, C = '%s')"% (C_param))

plt.show()

# + [markdown] id="YfEwAcpw9j8Q" colab_type="text"
# ## Linear Support Vector Machine

# + id="5rlK6ZG-eeOt" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 286} outputId="b6b0beab-aa82-484f-cd57-71a20654629c"
# Linear Support vector machine with only C Parameter 
for this_C in [1,3,5,10,40,60,80,100]:
    clf2 = LinearSVC(C=this_C, max_iter=50000).fit(X_train,Y_train)
    scoretrain2 = clf2.score(X_train,Y_train)
    scoretest2  = clf2.score(X_test,Y_test) 
    print("Linear SVM value of C:{}, training score :{:2f} , Test Score: {:2f} \n".format(this_C,scoretrain2,scoretest2))

# + id="IODe9HB-GrE4" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 350} outputId="1561c490-963e-4acf-efd2-d414ff7bf27c"
y_pred = clf2.predict(X_test)  
cm = confusion_matrix(y_pred, Y_test) 
fig, ax = plt.subplots(figsize=(8,5))
sns.heatmap(cm, annot = True, cmap = 'YlGnBu', fmt='g'); 
plt.title('Confusion matrix', fontweight='bold')
plt.ylabel('Actual label', fontweight='bold')
plt.xlabel('Predicted label', fontweight='bold'); 

# + [markdown] id="feBvhaGamx6z" colab_type="text"
# ## Apparently we got better scores with SVC where we defined the kernal as linear than with just LinearSVC
#
# The LinearSVC class is based on the liblinear library, which Uses a coordinate descent optimization algorithm for linear SVMs.
#
# 1. It does not support the kernel trick, but it scales almost linearly with the number of training instances and the number of features: its training time complexity is roughly O(m × n).
#
# The SVC class is based on the libsvm library, which implements the Sequential minimal optimization (SMO) algorithm for kernelized support vector machines (SVMs), supporting classification and regression, that supports the kernel trick.
#
# 1. The training time complexity is usually between O(m2 × n) and O(m3 × n).
# 2. LinearSVC is much faster than SVC(kernel="linear")
# 3. In this case, we got better scores in SVC than LinearSVC    

# + [markdown] id="9tATNHXEoGt2" colab_type="text"
# ## Support Vector Machine with Gaussian Kernel

# + id="09CdYs1noH6I" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 319} outputId="bcb1d31c-d9c3-4128-e837-4197f3745dac"
# SMV with RBF KERNAL AND ONLY C PARAMETER 

for this_C in [1,5,10,25,50,100]:
    clf3 = SVC(kernel='rbf',C=this_C).fit(X_train,Y_train)
    clf3train = clf3.score(X_train,Y_train)
    clf3test  = clf3.score(X_test,Y_test)
    print("SVM for Non Linear \n C:{} Training Score : {:2f} Test Score : {:2f}\n".format(this_C,clf3train,clf3test)) 

# + id="3TKLmGbMGy26" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 350} outputId="9202e158-8a25-4207-e5b0-baf92fd253f2"
y_pred = clf3.predict(X_test)  
cm = confusion_matrix(y_pred, Y_test) 
fig, ax = plt.subplots(figsize=(8,5))
sns.heatmap(cm, annot = True, cmap = 'YlGnBu', fmt='g'); 
plt.title('Confusion matrix', fontweight='bold')
plt.ylabel('Actual label', fontweight='bold')
plt.xlabel('Predicted label', fontweight='bold');  

# + id="I_P9lan9oUpT" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 281} outputId="936f41d2-738c-40ca-eac8-3ea0276d3a4b"
h = .02  # step size in the mesh
C_param = 1 # No of neighbours
# we create an instance of Neighbours Classifier and fit the data.
clf4 = SVC(kernel='rbf',C=C_param)
clf4.fit(x, y)

# Plot the decision boundary. For that, we will assign a color to each
# point in the mesh [x_min, x_max]x[y_min, y_max].
x_min, x_max = x[:, 0].min(), x[:, 0].max()
y_min, y_max = x[:, 1].min(), x[:, 1].max()
xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
Z = clf4.predict(np.c_[xx.ravel(), yy.ravel()])   # ravel to flatten the into 1D and c_ to concatenate 

# Put the result into a color plot
Z = Z.reshape(xx.shape)
plt.figure()
plt.pcolormesh(xx, yy, Z, cmap=cm_bright)
# Plot also the training points
plt.scatter(x[:, 0], x[:, 1], c=y, cmap=cm_dark, edgecolor='k', s=20)
plt.xlim(xx.min(), xx.max())
plt.ylim(yy.min(), yy.max())
plt.title("SVM Non Linear Classification (kernal = RBF, C = '%s')"% (C_param), fontweight='bold')

plt.show();

# + id="LMX1sYtrmVY0" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 1000} outputId="f26455cb-0f76-4bb6-df3e-26bd63d680bf"
# SVM WITH RBF KERNAL, C AND GAMMA HYPERPARAMTER 
for this_gamma in [.1,.5,.10,.25,.50,1]:
    for this_C in [1,5,7,10,15,25,50]:
        clf5 = SVC(kernel='rbf',C=this_C,gamma=this_gamma).fit(X_train,Y_train)
        clf5train = clf5.score(X_train,Y_train)
        clf5test  = clf5.score(X_test,Y_test)
        print("SVM for Non Linear \n Gamma: {} C:{} Training Score : {:2f} Test Score : {:2f}\n".format(this_gamma,this_C,clf5train,clf5test)) 

# + id="c9zD7bC3KBfD" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 350} outputId="d761ea52-f2e4-4111-c38e-29e7f40b1fdb"
y_pred = clf5.predict(X_test)  
cm = confusion_matrix(y_pred, Y_test) 
fig, ax = plt.subplots(figsize=(8,5))
sns.heatmap(cm, annot = True, cmap = 'YlGnBu', fmt='g'); 
plt.title('Confusion matrix', fontweight='bold')
plt.ylabel('Actual label', fontweight='bold')
plt.xlabel('Predicted label', fontweight='bold');  

# + id="n7RQbOiQ_NSm" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 281} outputId="4571c486-a261-4e59-bd4b-9b96010cd766"
h = .02  # step size in the mesh
C_param = 1 # No of neighbours
gamma_value = 0.1
# we create an instance of Neighbours Classifier and fit the data.
clf6 = SVC(kernel='rbf',C=C_param, gamma=gamma_value)
clf6.fit(x, y)

# Plot the decision boundary. For that, we will assign a color to each
# point in the mesh [x_min, x_max]x[y_min, y_max].
x_min, x_max = x[:, 0].min(), x[:, 0].max()
y_min, y_max = x[:, 1].min(), x[:, 1].max()
xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
Z = clf6.predict(np.c_[xx.ravel(), yy.ravel()])   # ravel to flatten the into 1D and c_ to concatenate 

# Put the result into a color plot
Z = Z.reshape(xx.shape)
plt.figure()
plt.pcolormesh(xx, yy, Z, cmap=cm_bright)
# Plot also the training points
plt.scatter(x[:, 0], x[:, 1], c=y, cmap=cm_dark, edgecolor='k', s=20)
plt.xlim(xx.min(), xx.max())
plt.ylim(yy.min(), yy.max())
plt.title("SVM Non Linear Classification (kernal = RBF, Gamma = {}, C = {})".format(gamma_value, C_param), fontweight='bold')

plt.show();

# + [markdown] id="fnET7CiDAt_j" colab_type="text"
# ## THE GRIDSEARCH !!!!
#
# Let's do something interesting,
# you can pass all the expected values of the hyperparameters and it'll return the best value which fits the model.

# + id="rAWAFnIRn6OS" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 235} outputId="0b7ff32c-55fa-4567-8e65-9b65992b0c07"
# grid search method 
param_grid = {'C': [1, 2, 3, 5, 7, 10, 15, 20, 25, 50, 100, 120, 150],
              'gamma': [.1, .2, .5,.10,.25,.50,1]}
GS = GridSearchCV(SVC(kernel='rbf'),param_grid,cv=5) 
GS.fit(X_train, Y_train)

# + id="SpT4OIreo97T" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 50} outputId="06a67430-fa0e-44d4-ca6e-e8f343dd5a53"
print("the parameters {} are the best.".format(GS.best_params_))
print("the best score is {:.2f}.".format(GS.best_score_)) 

# + [markdown] id="5RgAEtTq-Xvr" colab_type="text"
# ## Support Vector Regressor

# + id="06Zy5MvHgeNR" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="06eb1587-f4b6-4e03-cfee-a57fc02e35a8"
svr = SVR(kernel='linear', C=1, epsilon=.01).fit(X_train, Y_train)
print("{:.2f} is the accuracy of the SV Regressor".format(svr.score(X_train,Y_train))) 

# + [markdown] id="8lA7hyF7ng7n" colab_type="text"
# - SVM supports linear and nonlinear regression.
# - SVM Regression tries to fit as many instances as possible on the decision boundary while limiting margin violations.
# - The width of the decision boundary is controlled by a hyperparameter ϵ.

# + id="Da7k0nJ0pF6i" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="688b8886-5991-4b2c-e159-45cd18476e74"
# Kernalized SVM machine, by default kernel = 'rgf'
svr2 = SVR(degree=2,C=100,epsilon=.01).fit(X_train,Y_train)
print("{:.2f} is the accuracy of the SV Regressor".format(svr2.score(X_train, Y_train))) 

# + [markdown] id="Lcius6NcBxiW" colab_type="text"
# #Summary:
#
# - You learned how to implement different types of SVMs'
# - Visualized the SVM kernel plots
# - Confusion Matrix in form of heatmap
#

# + [markdown] id="n7oRzbcnWAc8" colab_type="text"
# ## Reference:
#
# - [SVM](https://scikit-learn.org/stable/modules/generated/sklearn.svm.SVC.html)

StarcoderdataJupyterScriptsDedupFiltered