# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="Aq7nFQZHm4Zl"
# Importing necessary libraries

# + id="qWU66LqPm8af"
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import confusion_matrix, accuracy_score

# + [markdown] id="umFTDaDlnPLU"
# Reading the dataset with Pandas after mounting the dataset in google colab

# + id="cCj4W_Q8nMEY"
#reading the dataset with a dataframe
df = pd.read_csv("final_preprocessed_dataset.csv")
df=df.drop('Unnamed: 0',1)

# + colab={"base_uri": "https://localhost:8080/", "height": 389} id="IDZMAnU1n-V4" executionInfo={"status": "ok", "timestamp": 1607024428551, "user_tz": -330, "elapsed": 1145, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/-nAR3so_GP8s/AAAAAAAAAAI/AAAAAAAAJus/3CfFQkqOuv8/s64/photo.jpg", "userId": "17934275643265308227"}} outputId="d5a76317-4127-4d4c-a31c-f92d3e5663f7"
#eye balling the data
df.head(5)

# + [markdown] id="CU6VA4KKqfP-"
# Dropping resource column and categorical APT group column

# + id="MUc7p6mpqj-w"
df = df.drop('Resource', 1)
df =df.drop('APT Group',1)

# + id="KG32i0KwonId" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1607024439041, "user_tz": -330, "elapsed": 1093, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/-nAR3so_GP8s/AAAAAAAAAAI/AAAAAAAAJus/3CfFQkqOuv8/s64/photo.jpg", "userId": "17934275643265308227"}} outputId="b799a832-9bfc-40e5-86ca-ab9afa59be7f"
#eye balling information about our dataset
df.head(5)
df.info()

# + [markdown] id="uw80LLnIpAr2"
# Sklearn Train Test Split Method : Splitting the data into training and testing data.

# + id="hmtvadQIozzG"
#the labelled column is Apt Group and the rest is features
label = np.array(df['APTGroup'])
features = df.drop('APTGroup',1)

# + colab={"base_uri": "https://localhost:8080/", "height": 355} id="tlfcRvnTqQj_" executionInfo={"status": "ok", "timestamp": 1606807513486, "user_tz": -330, "elapsed": 2281, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/-nAR3so_GP8s/AAAAAAAAAAI/AAAAAAAAJus/3CfFQkqOuv8/s64/photo.jpg", "userId": "17934275643265308227"}} outputId="38a0fe69-11cb-40a1-e4b1-ed11ad6de1f4"
features.head()


# + colab={"base_uri": "https://localhost:8080/"} id="2FppczNYwFfe" executionInfo={"status": "ok", "timestamp": 1606807513491, "user_tz": -330, "elapsed": 2275, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/-nAR3so_GP8s/AAAAAAAAAAI/AAAAAAAAJus/3CfFQkqOuv8/s64/photo.jpg", "userId": "17934275643265308227"}} outputId="d4b78da3-3464-4dbe-b883-846048c78901"
label

# + id="tFjI9r5CpP0L"
#choosing a 70-30 split to test out the performance
from sklearn.model_selection import train_test_split
seed =50
X_train, X_test, y_train, y_test = train_test_split(features,label,test_size=0.30, random_state = seed)

# + id="hhVRpS2SphSP" colab={"base_uri": "https://localhost:8080/", "height": 355} executionInfo={"status": "ok", "timestamp": 1606807513899, "user_tz": -330, "elapsed": 2658, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/-nAR3so_GP8s/AAAAAAAAAAI/AAAAAAAAJus/3CfFQkqOuv8/s64/photo.jpg", "userId": "17934275643265308227"}} outputId="660da4ae-b867-4ea1-97ac-23b27ae17015"
X_train.head(5)

# + id="b-v3szuWppsQ" colab={"base_uri": "https://localhost:8080/", "height": 355} executionInfo={"status": "ok", "timestamp": 1606807513913, "user_tz": -330, "elapsed": 2655, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/-nAR3so_GP8s/AAAAAAAAAAI/AAAAAAAAJus/3CfFQkqOuv8/s64/photo.jpg", "userId": "17934275643265308227"}} outputId="92cb8b37-eeed-4dd5-8dbb-9292d5b50110"
X_test.head(5)

# + [markdown] id="ZqF_BNh5qJAr"
# Building the Random Forest Classifier

# + id="enqRkSj5p4rL"
rf_classifier = RandomForestClassifier(
                      min_samples_leaf=50,
                      n_estimators=150,
                      bootstrap=True,
                      oob_score=True,
                      n_jobs=-1,
                      random_state=seed,
                      max_features='auto')

# + id="tv6uDVvcqO5k" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1606807513918, "user_tz": -330, "elapsed": 2643, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/-nAR3so_GP8s/AAAAAAAAAAI/AAAAAAAAJus/3CfFQkqOuv8/s64/photo.jpg", "userId": "17934275643265308227"}} outputId="493749e0-a9f2-4d4a-e5f6-1ad5868a02be"
rf_classifier.fit(X_train,y_train)

# + id="g3xVIV_PqRZz"
y_pred = rf_classifier.predict(X_test)

# + id="teXAXkmQqxjH" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1606807514291, "user_tz": -330, "elapsed": 3001, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/-nAR3so_GP8s/AAAAAAAAAAI/AAAAAAAAJus/3CfFQkqOuv8/s64/photo.jpg", "userId": "17934275643265308227"}} outputId="41fda03e-4dff-464c-e443-cfa52f0506bd"
print(y_pred)

# + [markdown] id="eiFEQe-mq2TT"
# Confusion Matrix

# + id="v7hasH0xq4Tc" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1606807514293, "user_tz": -330, "elapsed": 2987, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/-nAR3so_GP8s/AAAAAAAAAAI/AAAAAAAAJus/3CfFQkqOuv8/s64/photo.jpg", "userId": "17934275643265308227"}} outputId="abc875d5-d144-4ef3-9045-8ea8ae218187"
cm = confusion_matrix(y_test, y_pred)
print(cm)
accuracy_score(y_test, y_pred)

# + [markdown] id="5MxpXYgkrDQP"
# # Observation:
#
# Accuracy acquired about 58% accuracy. Needs more hyperparameter tuning to increase the accuracy. Maybe contributing other factors along the way too can lead to better accuracy.

# + [markdown] id="OSZYPJubdmtM"
# ## Model Attempt#2 : Hyper Parameter tunning

# + id="rrHdIN9xdWB9"
"""
rf_classifier = RandomForestClassifier(
                      min_samples_leaf=50,
                      n_estimators=150,
                      bootstrap=True,
                      oob_score=True,
                      n_jobs=-1,
                      random_state=seed,
                      max_features='auto')
                      """
rf_classifier = RandomForestClassifier(
                      min_samples_leaf=3,
                      n_estimators=300,
                      bootstrap=True,
                      oob_score=True,
                      n_jobs=-1,
                      random_state=seed,
                      max_features='auto')

# + colab={"base_uri": "https://localhost:8080/"} id="c35NvrKExAw3" executionInfo={"status": "ok", "timestamp": 1606807515575, "user_tz": -330, "elapsed": 4250, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/-nAR3so_GP8s/AAAAAAAAAAI/AAAAAAAAJus/3CfFQkqOuv8/s64/photo.jpg", "userId": "17934275643265308227"}} outputId="bbb7e314-255b-4424-d423-e399a661455d"
rf_classifier.fit(X_train, y_train)
y_pred = rf_classifier.predict(X_test)
print("accuracy ", accuracy_score(y_test, y_pred) )

# + [markdown] id="z6N1hMWE01mQ"
# #Cross validation 

# + colab={"base_uri": "https://localhost:8080/", "height": 355} id="mSjJV2kb1J8n" executionInfo={"status": "ok", "timestamp": 1606807515580, "user_tz": -330, "elapsed": 4243, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/-nAR3so_GP8s/AAAAAAAAAAI/AAAAAAAAJus/3CfFQkqOuv8/s64/photo.jpg", "userId": "17934275643265308227"}} outputId="78f059c7-4934-467d-88ab-eb0a40a74fbf"
df.head(5)

# + id="TwRYQGf5Nh5R" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1606807541045, "user_tz": -330, "elapsed": 29697, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/-nAR3so_GP8s/AAAAAAAAAAI/AAAAAAAAJus/3CfFQkqOuv8/s64/photo.jpg", "userId": "17934275643265308227"}} outputId="a5a1054d-cac5-4ce7-f4bf-cfa4de7de2f1"
 from sklearn.model_selection import ShuffleSplit
 ss = ShuffleSplit(n_splits=20, test_size=0.30)
accuracy=0
for train_indices, test_indices in ss.split(df):
  #print("Train")
  #print(X_train)
  #print("Test")
  #print(X_test)
  
  train = df.iloc[train_indices]
  y_train= np.array(train['APTGroup'])
  X_train= train.drop('APTGroup',1)
  

  test = df.iloc[test_indices]
  y_test= np.array(train['APTGroup'])
  X_test= train.drop('APTGroup',1)
  
  rf_classifier.fit(X_train, y_train)
  y_pred = rf_classifier.predict(X_test)
  print("accuracy ", accuracy_score(y_test, y_pred) )
  accuracy+=accuracy_score(y_test, y_pred)

print(accuracy/20)


# + [markdown] id="UM_EDGT9QtyD"
# # Observation 1:
#
# Accuracy acquired : 86.62 % for n_splits=20 \\

# + [markdown] id="xAgsk_Uo22E_"
# # Another method for cross validation: cross_val_score
#

# + colab={"base_uri": "https://localhost:8080/"} id="4dPMhSjd2-rM" executionInfo={"status": "ok", "timestamp": 1606807570615, "user_tz": -330, "elapsed": 59258, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/-nAR3so_GP8s/AAAAAAAAAAI/AAAAAAAAJus/3CfFQkqOuv8/s64/photo.jpg", "userId": "17934275643265308227"}} outputId="c3cd020b-3fa7-413c-e28a-09b29fa3bd00"
from sklearn.model_selection import cross_val_score
from sklearn.ensemble import RandomForestClassifier
print(cross_val_score(rf_classifier, features, label, cv=20, scoring ='accuracy').mean())

# + [markdown] id="XsEcvI0T3Jnx"
# # Observation 2:
#
# Accuracy acquired : 84.2 % for n_splits=20 \\

# + [markdown] id="Byu3lM7NpFhF"
# ## Important Feature Calculation

# + id="fUD2NR27dgbu" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1606807571505, "user_tz": -330, "elapsed": 60121, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/-nAR3so_GP8s/AAAAAAAAAAI/AAAAAAAAJus/3CfFQkqOuv8/s64/photo.jpg", "userId": "17934275643265308227"}} outputId="e06727ff-1004-4f4e-b47d-8211577ae913"
rf_classifier.fit(X_train,y_train)

# + colab={"base_uri": "https://localhost:8080/", "height": 293} id="I3ptheS5pQOS" executionInfo={"status": "ok", "timestamp": 1606807571507, "user_tz": -330, "elapsed": 60108, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/-nAR3so_GP8s/AAAAAAAAAAI/AAAAAAAAJus/3CfFQkqOuv8/s64/photo.jpg", "userId": "17934275643265308227"}} outputId="47a8445a-96b9-47a6-dbf0-c2df52e6596c"
df.head(3)

# + id="XbJvnoDGq_Ub"
pd.set_option("display.max.rows", None)

# + id="xiQQNyVprHHd" colab={"base_uri": "https://localhost:8080/", "height": 1000} executionInfo={"status": "ok", "timestamp": 1606807571983, "user_tz": -330, "elapsed": 60543, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/-nAR3so_GP8s/AAAAAAAAAAI/AAAAAAAAJus/3CfFQkqOuv8/s64/photo.jpg", "userId": "17934275643265308227"}} outputId="4514c9f1-73cb-443a-e6a7-1b1f3824b29f"
# Important Features
feature_importance = pd.DataFrame(rf_classifier.feature_importances_,
                                   index = X_train.columns,
                                    columns=['importance']).sort_values('importance',ascending=False)
feature_importance
# summarize feature importance
#for i,v in enumerate(importance):
 #   print('Feature: %0d, Score: %.5f' % (i,v))

# + [markdown] id="2yeqzJAJttxh"
# Exporting the results into an excel file for better view

# + id="zXxomxYLtb6v"
excel_data = feature_importance.copy()

# + id="9ZATk1qjt9HR"
excel_data.to_csv('important_features.csv', index=True)

# + colab={"base_uri": "https://localhost:8080/"} id="T0T43YIJu458" executionInfo={"status": "ok", "timestamp": 1606807571994, "user_tz": -330, "elapsed": 60527, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/-nAR3so_GP8s/AAAAAAAAAAI/AAAAAAAAJus/3CfFQkqOuv8/s64/photo.jpg", "userId": "17934275643265308227"}} outputId="1859f38f-3daf-40a7-f07f-9869fc97590b"
# %ls

# + colab={"base_uri": "https://localhost:8080/"} id="J_FRBzYWu4RO" executionInfo={"status": "ok", "timestamp": 1606807571996, "user_tz": -330, "elapsed": 60520, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/-nAR3so_GP8s/AAAAAAAAAAI/AAAAAAAAJus/3CfFQkqOuv8/s64/photo.jpg", "userId": "17934275643265308227"}} outputId="afbf6f4e-d558-4618-a28d-bcc52cc64692"
#checking with shell command if there was a proper write in the new file created
# ! cat important_features.csv

StarcoderdataJupyterScriptsDedupFiltered