# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/parthrjpt/EAS555_Projects/blob/main/ProblemSET2_555.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + id="kyaCwS_t2S1V"
# %matplotlib inline

import pandas as pd
import numpy as np
import os
import cv2
import matplotlib.pyplot as plt
import random
import tensorflow as tf

# + colab={"base_uri": "https://localhost:8080/"} id="eHXWYNpP2HhJ" outputId="5bbe75a3-c5c9-4875-f32a-ccf8cc4d8b31"
from keras.datasets import mnist
(X_train, y_train), (X_test, y_test) = mnist.load_data()
print(X_train.shape, y_train.shape,X_test.shape, y_test.shape)

# + id="pe3IN5N6AWYQ"
X_train = X_train.reshape(60000,784)
X_test = X_test.reshape(10000,784)



# + id="cWiKm6G-Om49"
from sklearn.model_selection import train_test_split
from sklearn.decomposition import PCA

from sklearn import preprocessing
from sklearn.preprocessing import StandardScaler

from sklearn import svm
from sklearn.model_selection import GridSearchCV
from sklearn.gaussian_process.kernels import DotProduct
from sklearn.svm import SVC
from sklearn.pipeline import Pipeline

X_train_proc = preprocessing.scale(X_train)
X_test_proc = preprocessing.scale(X_test)

from sklearn.preprocessing import MinMaxScaler
scaling = MinMaxScaler(feature_range=(-1,1)).fit(X_train_proc)
X_train_proc = scaling.transform(X_train_proc)
X_test_proc = scaling.transform(X_test_proc)

pca = PCA(n_components=0.85)
X_train_proc= pca.fit_transform(X_train_proc)
X_test_proc = pca.fit_transform(X_test_proc)

#preproc = [('scaler', MinMaxScaler(feature_range=(-1,1)).fit(X_train)),('pca', PCA()),('clf',SVC(kernel='linear'))]

#pipeline = Pipeline(preproc)
#parameters = {'kernel':('linear', 'rbf'), 'C':[1,10]}

#grid = GridSearchCV(pipeline, param_grid=parameters, cv=5)
#grid.fit(X_train, y_train)

#Create a svm Classifier

from sklearn.model_selection import cross_validate
from sklearn.metrics import recall_score

clf = SVC()
# clf=[clf1, clf2]
param_grid = {
      'kernel': ['linear','rbf'],
      'C': [0.1,0.5],
      'gamma': [0.0001, 0.001, 0.01]
    }
grid=GridSearchCV(clf, param_grid = param_grid , cv = 5, scoring = 'accuracy')
grid_clf = grid.fit( X_train_proc, y_train)

#scoring = ['precision_macro', 'accuracy_macro']
#scores = cross_validate(clf, X_train_proc, y_train, scoring=scoring)
#Train the model using the training sets
#Based on a single run using svm's predict , the ccuracy was around 90%. The overall run for 60k training data with hyperparamtee tuning is timeconsuming
# Therfore, this could not be computed.
#While extremely, slow the final accuracy is around 95.92%

# + id="G4ZlBpaBDVV_"
y_pred = cross_val_predict(clf, X_test_proc, y_test, cv=3)

# + colab={"base_uri": "https://localhost:8080/"} id="xVZ_L1es7zQI" outputId="44045b30-88cb-4292-9a27-59fa035a169a"
i=0
count=0
while i<10000:
  if(y_test[i]==y_pred[i]):
    count=count+1
  i=i+1
print("Accuracy: ",(count/len(y_test)*100),'%')

# + [markdown] id="I0ku_zhsTu0O"
# \begin{align*}
# \text{minimize  } \ w^T.w+C \sum_{n=1}^{N} \xi_{i}
# \\
# \text{subject to } y_i.(w^T.x_i) \geq {1-\xi_{i}} \text{ and } \xi_i \geq 0 \text{ for i }=1,...,N
# \end{align*}

# + [markdown] id="lGngOHHjT_Xv"
# \begin{align}
# {\mathcal{L}} = \ {\overrightarrow{w}}^T\overrightarrow{w} + C \sum_{i=1}^N\xi_i + \Sigma_i\alpha_i(y_i.w^Tx_i -1+\xi_i)
# \end{align}
# <br><br>
# Lagrange multipliers $\alpha \geq 0$<br>
# Inequality constraints $y_i(w^Tx_i) \geq 1- \xi_i$ and $\xi_i \geq 0$ for $i=1,...,N$
# <br>
# Therefore,
# <br>
# $y_i(w^Tx_i) - 1 \leq \xi_i$ 
# <br><br>
#
# \begin{align}
# {\mathcal{L}} = \ {\overrightarrow{w}}^T\overrightarrow{w} + C \sum_{i=1}^N\xi_i - \Sigma_i\alpha_i(y_i.w^Tx_i - 1+ \xi_i)
# \end{align}
#
# \begin{align}
# \\
#   \frac{\partial_{\mathcal{L}}}{\partial_w} = w - \sum_{i=1}^N\alpha_i y_i x_i = 0 \implies w = \sum_{i=1}^N\alpha_i y_i \overrightarrow{x_i}
#   \\
#   \frac{\partial_{\mathcal{L}}}{\partial_\xi} = 0 \implies C - \alpha_i= 0
#   \\ 
#   \implies 0 \leq \alpha_i \leq C 
#   \end{align}
#
# \begin{align}
# \text {Substituting } w = \sum_{i=1}^N\alpha_i y_i \overrightarrow{x_i}, C=\alpha_i 
# \\
# \text { into Lagrange function, we get the dual problem of maximizing: }
# \end{align}
#
# \begin{align}
# {\mathcal{L}} = \ w^T \sum_{i=1}^N\alpha_i y_i \overrightarrow{x_i} + \alpha_i\sum_{i=1}^N\xi_i + \sum_{i=1}^N\alpha_i(y_i.w^Tx_i -1+\xi_i)
# \\= \ w^T \sum_{i=1}^N \alpha_i y_i \overrightarrow{x_i} + \alpha_i\sum_{i=1}^N\xi_i  +\sum_{i=1}^N \alpha_i - w^T \sum_{i=1}^N \alpha_i y_i \overrightarrow{x_i} - \sum_{i=1}^N \alpha_i \xi_i
# \end{align}
#
# \begin{align}
# \\=\alpha_i\sum_{i=1}^N \xi_i -\sum_{i=1}^N \alpha_i  +\sum_{i=1}^N \alpha_i \sum_{i=1}^N \xi_i 
# \end{align}
#

# + [markdown] id="BfqpKXNpgTH8"
#
# The primal margin is :
#  $\gamma = \frac{1}{\sqrt{w^T.w +C \sum_{i=1}^{N} \xi_i }}$
#  
# The dual margin is :
# $\gamma = \frac{1}{\sqrt{\alpha_i + \sum_{i=1}^{N} \alpha_i \sum_{i=1}^{N} \xi_i }}$
#

# + [markdown] id="4KKOfHcfi_rF"
# ### Benefits of maximizing the margin
#
# * Maximizing the margin regularizes SVM weights that helps avoid overfitting.
# * A large margin would help reduce the functional space.
# * Better generalization of small sample spaces
#

# + [markdown] id="g_qU5FMYjNcH"
# #### Characterize the support vectors
#
# * Margin boundaries:
# \begin{align}
# w^Tx + C = 1 \text{ and } w^Tx + C = -1
# \end{align}
#
# * Margin region (0 < ξ < 1)

# + [markdown] id="GCBaRkfzjG7C"
# #### Benefits of solving the dual problem instead of the primal problem
#
# * Solving the primal problem gives w but there is no insight on αi. 
# * Dual representation provides lower bound to primal problem.
# * Dual representation can be solved using Kernel trick

StarcoderdataJupyterScriptsDedupFiltered