# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import pandas as pd 
import numpy as np
import sys
from tqdm import tqdm

from preprocessing_ml_table import *

from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler
from sklearn.feature_selection import SelectKBest, chi2
from sklearn.decomposition import PCA

from sklearn.naive_bayes import GaussianNB
from sklearn import svm
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import AdaBoostClassifier

from xgboost import XGBClassifier

np.set_printoptions(threshold=sys.maxsize)
# -

# ## Reading the Dataset

df_surtur = pd.read_csv('./datasets_of_interest/super_final_dataset_surtur.csv')
df_surtur.head()

print(np.array(df_surtur.columns))

# ## Machine Learning

# ### Summary Table

# | Variation                   | Accuracy alone | Accuracy with variations above |
# |-----------------------------|----------------|--------------------------------|
# | Baseline (no embeddings)    | N/A            | 83.11%                         |
# | Lexical                     | N/A            | <mark>75.48%</mark>            |
# | Host-based                  | 65.70%         | <mark>80.02%</mark>            |
# | Content-based               | 76.30%         | <mark>83.11%</mark>            |
# | lngfrm all embs             | 76.38%         | 83.23%                         |
# | lngfrm embs PCA 10          | 75.41%         | <mark>83.73%</mark>            |
# | lngfrm embs PCA 25          | 75.24%         | 82.41%                         |
# | lngfrm embs PCA 50          | 75.25%         | 82.09%                         |
# | lngfrm feat selection       | 76.43%         | 83.29%                         |
# | img all embs                | 60.27%         | 82.24%                         |
# | img embs PCA 10             | 62.57%         | <mark>83.69%</mark>            |
# | img embs PCA 25             | 61.73%         | 83.46%                         |
# | img embs PCA 50             | 61.16%         | 82.82%                         |
# | img feat selection          | 61.29%         | 83.62%                         |
# | content all embeddings      | 67.63%         | 81.89%                         |
# | content embs PCA 10         | 67.71%         | <mark>83.74%</mark>            |
# | content embs PCA 25         | 68.11%         | 83.58%                         |
# | content embs PCA 50         | 67.76%         | 83.06%                         |
# | content feat selection      | 66.57%         | 83.59%                         |
# | BERT url embs all           | 76.62%         | 82.47%                         |
# | BERT url embs PCA 10        | 72.97%         | <mark>83.79%</mark>            |
# | BERT url embs PCA 25        | 73.24%         | 83.57%                         |
# | BERT url embs PCA 50        | 74.08%         | 82.99%                         |
# | BERT url feat selection     | 75.12%         | 83.64%                         |
# | Best Score (Feat selection) | N/A            | <mark>84.27%</mark>            |
# | Best Score (PCA)            | N/A            | <mark>83.79%</mark>           |

# ### All Features without embeddings

# +
baseline_feats = []
for feature in df_surtur.columns:
    if all(value not in feature for value in ['emb', 'mobnet', 'img_mean', 'url_emb_mean']):
        baseline_feats.append(feature)
        
make_classification(baseline_feats, df_surtur, tuning=True)
# -

# ### Lexical Features

feats_to_drop = ['label', 'content', 'hostname', 'url', 'js', 
                    'domain', 'google_is_safe', 'ip_address']

# +
lexical_features = ['has_IP_in_url', 'having_@_in_url', 'length_hostname', 'number_subdomains', 
                    'number_underscores', 'ratio_digits_hostname', 'ratio_digits_url', 'tld', 
                    'url_len', 'num_semicolons', 'num_zeros', 'num_spaces', 'num_hyphens', 
                    'num_@s', 'num_queries', 'num_ampersands', 'num_equals', 'domain_len',
                    'hex_len', 'has_hex', 'unique_url_chars', 'unique_url_nums', 'unique_url_letters',
                    'ratio_let_chars', 'ratio_nums_chars', 'num_?s']

make_classification(lexical_features + feats_to_drop, df_surtur)
# -

# ### Host-based Features

# +
host_feats = ['https', 'who_is', 'is_in_alexa', 'location']

make_classification(host_feats + feats_to_drop, df_surtur)
make_classification(lexical_features + host_feats + feats_to_drop, df_surtur)
# -

# ### Content-based Features

# +
content_feats = ['js_len', 'js_ref', 'js_array_len_avg', 'js_array_len_max',
                 'content_len', 'num_js_func_calls', 'malicious_func_count','total_url_count', 
                 'ext_url_count', 'dcd_color_1', 'dcd_color_2', 'dcd_color_3', 'dcd_color_4', 'dcd_color_5']

make_classification(content_feats + feats_to_drop, df_surtur)
make_classification(lexical_features + host_feats + feats_to_drop + content_feats, df_surtur)
# -

# ## URL Embeddings from the longformer 
#
# https://huggingface.co/transformers/model_doc/longformer.html

# ### All embeddings

# +
url_embeddings = pd.read_csv('./datasets_of_interest/url_feats_sliding_window_longformer.csv')
scaled_embs = scale_features(url_embeddings)
url_embs_all, url_embs_all_columns = make_columns_for_embs(scaled_embs, 'url_emb_all')
url_embs_all_columns = url_embs_all_columns

df_surtur = pd.concat([df_surtur, url_embs_all], axis=1)
# -

make_classification(url_embs_all_columns + feats_to_drop, df_surtur)
make_classification(url_embs_all_columns + feats_to_drop + lexical_features 
                    + host_feats + content_feats, df_surtur)

# ### PCA 10

# +
pca_10 = perform_pca(10, scaled_embs)
url_embs_pca_10, url_embs_pca_10_columns = make_columns_for_embs(pca_10, 'url_pca_10')
# url_embs_pca_10_columns = url_embs_pca_10_columns + ['url_emb_mean', 'url_emb_mean_red']

df_surtur = pd.concat([df_surtur, url_embs_pca_10], axis=1)
# -

make_classification(url_embs_pca_10_columns + feats_to_drop, df_surtur)
make_classification(url_embs_pca_10_columns + feats_to_drop + lexical_features 
                    + host_feats + content_feats, df_surtur)

# ### PCA 25

# +
pca_25 = perform_pca(25, scaled_embs)
url_embs_pca_25, url_embs_pca_25_columns = make_columns_for_embs(pca_25, 'url_pca_25')
# url_embs_pca_25_columns = url_embs_pca_25_columns + ['url_emb_mean', 'url_emb_mean_red']

df_surtur = pd.concat([df_surtur, url_embs_pca_25], axis=1)
# -

make_classification(url_embs_pca_25_columns + feats_to_drop, df_surtur)
make_classification(url_embs_pca_25_columns + feats_to_drop + lexical_features 
                    + host_feats + content_feats, df_surtur)

# ### PCA 50

# +
pca_50 = perform_pca(50, scaled_embs)
url_embs_pca_50, url_embs_pca_50_columns = make_columns_for_embs(pca_50, 'url_pca_50')
# url_embs_pca_50_columns = url_embs_pca_50_columns + ['url_emb_mean', 'url_emb_mean_red']

df_surtur = pd.concat([df_surtur, url_embs_pca_50], axis=1)
# -

make_classification(url_embs_pca_50_columns + feats_to_drop, df_surtur)
make_classification(url_embs_pca_50_columns + feats_to_drop + lexical_features 
                    + host_feats + content_feats, df_surtur)

# ### Feature Selection

# +
url_emb_feat_sel = ['url_emb_mean', 'url_emb_mean_red', 'url_emb_0', 
                     'url_emb_1', 'url_emb_2', 'url_emb_3', 'url_emb_4', 
                     'url_emb_5', 'url_emb_6', 'url_emb_7', 'url_emb_8', 
                     'url_emb_9', 'url_emb_10', 'url_emb_11', 'url_emb_12', 
                     'url_emb_13', 'url_emb_14', 'url_emb_15', 'url_emb_16', 
                     'url_emb_17', 'url_emb_18', 'url_emb_19']

make_classification(url_emb_feat_sel + feats_to_drop, df_surtur)
make_classification(url_emb_feat_sel + feats_to_drop + lexical_features + host_feats + content_feats, df_surtur)
# -

# ### Best one: 
#
# The best set to continue with is the PCA 10 components set

# ## Image Embeddings

# ### All embeddings

# +
img_embeddings = pd.read_csv('./datasets_of_interest/img_features_adjusted.csv')
scaled_embs = scale_features(np.array(img_embeddings))
img_embs_all, img_embs_all_columns = make_columns_for_embs(scaled_embs, 'img_emb_all')

df_surtur = pd.concat([df_surtur, img_embs_all], axis=1)
# -

make_classification(img_embs_all_columns + feats_to_drop, df_surtur)
make_classification(img_embs_all_columns + feats_to_drop + lexical_features 
                    + host_feats + content_feats + url_embs_pca_10_columns, df_surtur)

# ### PCA 10

# +
pca_10 = perform_pca(10, scaled_embs)
img_embs_pca_10, img_embs_pca_10_columns = make_columns_for_embs(pca_10, 'img_pca_10')

df_surtur = pd.concat([df_surtur, img_embs_pca_10], axis=1)
# -

make_classification(img_embs_pca_10_columns + feats_to_drop, df_surtur)
make_classification(img_embs_pca_10_columns + feats_to_drop + lexical_features 
                    + host_feats + content_feats + url_embs_pca_10_columns, df_surtur)

# ### PCA 25

# +
pca_25 = perform_pca(25, scaled_embs)
img_embs_pca_25, img_embs_pca_25_columns = make_columns_for_embs(pca_25, 'img_pca_25')

df_surtur = pd.concat([df_surtur, img_embs_pca_25], axis=1)
# -

make_classification(img_embs_pca_25_columns + feats_to_drop, df_surtur)
make_classification(img_embs_pca_25_columns + feats_to_drop + lexical_features 
                    + host_feats + content_feats + url_embs_pca_10_columns, df_surtur)

# ### PCA 50

# +
pca_50 = perform_pca(50, scaled_embs)
img_embs_pca_50, img_embs_pca_50_columns = make_columns_for_embs(pca_50, 'img_pca_50')
# img_embs_pca_50_columns = img_embs_pca_50_columns + ['img_emb_mean', 'img_emb_mean_red']

df_surtur = pd.concat([df_surtur, img_embs_pca_50], axis=1)
# -

make_classification(img_embs_pca_50_columns + feats_to_drop, df_surtur)
make_classification(img_embs_pca_50_columns + feats_to_drop + lexical_features 
                    + host_feats + content_feats + url_embs_pca_10_columns, df_surtur)

# ### Feature Selection

# +
img_mobnet_embeddings = ['img_mean', 'img_mean_red', 'image_mobnet_0', 
                         'image_mobnet_1', 'image_mobnet_2', 'image_mobnet_3', 
                         'image_mobnet_4', 'image_mobnet_5', 'image_mobnet_6', 'image_mobnet_7', 
                         'image_mobnet_8', 'image_mobnet_9', 'image_mobnet_10', 'image_mobnet_11', 
                         'image_mobnet_12', 'image_mobnet_13', 'image_mobnet_14', 'image_mobnet_15',
                         'image_mobnet_16', 'image_mobnet_17', 'image_mobnet_18', 'image_mobnet_19']

make_classification(img_mobnet_embeddings + feats_to_drop, df_surtur)
make_classification(img_mobnet_embeddings + feats_to_drop + lexical_features + host_feats
                    + content_feats + url_embs_pca_10_columns, df_surtur)
# -

# ### Best one: 
#
# The best one is once again PCA with 10 components

# ## Content Embeddings

# ### All embeddings

# +
content_embeddings = pd.read_csv('./datasets_of_interest/content_feats_code_bert.csv')
scaled_embs = scale_features(np.array(content_embeddings))
content_embs_all, content_embs_all_columns = make_columns_for_embs(scaled_embs, 'content_emb_all')

df_surtur = pd.concat([df_surtur, content_embs_all], axis=1)
# -

make_classification(content_embs_all_columns + feats_to_drop, df_surtur)
make_classification(content_embs_all_columns + feats_to_drop + lexical_features 
                    + host_feats + content_feats 
                    + url_embs_pca_10_columns + img_embs_pca_10_columns, df_surtur)

# ### PCA 10

# +
pca_10 = perform_pca(10, scaled_embs)
content_embs_pca_10, content_embs_pca_10_columns = make_columns_for_embs(pca_10, 'content_pca_10')

df_surtur = pd.concat([df_surtur, content_embs_pca_10], axis=1)
# -

make_classification(content_embs_pca_10_columns + feats_to_drop, df_surtur)
make_classification(content_embs_pca_10_columns + feats_to_drop + lexical_features 
                    + host_feats + content_feats 
                    + url_embs_pca_10_columns + img_embs_pca_10_columns, df_surtur)

# ### PCA 25

# +
pca_25 = perform_pca(25, scaled_embs)
content_embs_pca_25, content_embs_pca_25_columns = make_columns_for_embs(pca_25, 'content_pca_25')

df_surtur = pd.concat([df_surtur, content_embs_pca_25], axis=1)
# -

make_classification(content_embs_pca_25_columns + feats_to_drop, df_surtur)
make_classification(content_embs_pca_25_columns + feats_to_drop + lexical_features 
                    + host_feats + content_feats
                    + url_embs_pca_10_columns + img_embs_pca_10_columns, df_surtur)

# ### PCA 50

# +
pca_50 = perform_pca(50, scaled_embs)
content_embs_pca_50, content_embs_pca_50_columns = make_columns_for_embs(pca_50, 'content_pca_50')

df_surtur = pd.concat([df_surtur, content_embs_pca_50], axis=1)
# -

make_classification(content_embs_pca_50_columns + feats_to_drop, df_surtur)
make_classification(content_embs_pca_50_columns + feats_to_drop + lexical_features 
                    + host_feats + content_feats 
                    + url_embs_pca_10_columns + img_embs_pca_10_columns, df_surtur)

# ## Feature Selection

# +
content_embeddings = ['content_emb_0', 'content_emb_1', 'content_emb_2', 
                      'content_emb_3', 'content_emb_4', 'content_emb_5', 
                      'content_emb_6', 'content_emb_7', 'content_emb_8', 
                      'content_emb_9', 'content_emb_10', 'content_emb_11', 
                      'content_emb_12', 'content_emb_13', 'content_emb_14', 
                      'content_emb_15', 'content_emb_16', 'content_emb_17', 
                      'content_emb_18', 'content_emb_19']

make_classification(content_embeddings + feats_to_drop, df_surtur)
make_classification(url_embs_pca_10_columns + img_embs_pca_10_columns 
                    + feats_to_drop + lexical_features + host_feats
                    + content_feats + content_embeddings, df_surtur)
# -

# ### Best one:
#
# PCA 10 once again

# ### DCD

# +
dcd_feats = ['dcd_color_1', 'dcd_color_2', 'dcd_color_3', 'dcd_color_4', 'dcd_color_5']

make_classification(dcd_feats + feats_to_drop, df_surtur)
# -

# ## Additional embeddings from BERT

# ### All embeddings

# +
bert_url_embeddings = pd.read_csv('./datasets_of_interest/url_feats_sliding_window.csv')
scaled_embs = scale_features(np.array(bert_url_embeddings))
bert_url_embs_all, bert_url_embs_all_columns = make_columns_for_embs(scaled_embs, 'bert_url_emb_all')

df_surtur = pd.concat([df_surtur, bert_url_embs_all], axis=1)
# -

make_classification(bert_url_embs_all_columns + feats_to_drop, df_surtur)
make_classification(bert_url_embs_all_columns + content_feats + feats_to_drop + lexical_features 
                    + host_feats + url_embs_pca_10_columns + img_embs_pca_10_columns 
                    + content_embs_pca_10_columns, df_surtur)

# ### PCA 10

# +
pca_10 = perform_pca(10, scaled_embs)
bert_url_embs_pca_10, bert_url_embs_pca_10_columns = make_columns_for_embs(pca_10, 'bert_url_pca_10')

df_surtur = pd.concat([df_surtur, bert_url_embs_pca_10], axis=1)
# -

make_classification(bert_url_embs_pca_10_columns + feats_to_drop, df_surtur)
make_classification(bert_url_embs_pca_10_columns + feats_to_drop + lexical_features 
                    + host_feats + content_feats 
                    + url_embs_pca_10_columns + img_embs_pca_10_columns + content_embs_pca_10_columns, df_surtur)

# ### PCA 25

# +
pca_25 = perform_pca(25, scaled_embs)
bert_url_embs_pca_25, bert_url_embs_pca_25_columns = make_columns_for_embs(pca_25, 'bert_url_pca_25')

df_surtur = pd.concat([df_surtur, bert_url_embs_pca_25], axis=1)
# -

make_classification(bert_url_embs_pca_25_columns + feats_to_drop, df_surtur)
make_classification(bert_url_embs_pca_25_columns + feats_to_drop + lexical_features 
                    + host_feats + content_feats
                    + url_embs_pca_10_columns + img_embs_pca_10_columns + content_embs_pca_10_columns, df_surtur)

# ### PCA 50

# +
pca_50 = perform_pca(50, scaled_embs)
bert_url_embs_pca_50, bert_url_embs_pca_50_columns = make_columns_for_embs(pca_50, 'bert_url_pca_50')

df_surtur = pd.concat([df_surtur, bert_url_embs_pca_50], axis=1)
# -

make_classification(bert_url_embs_pca_50_columns + feats_to_drop, df_surtur)
make_classification(bert_url_embs_pca_50_columns + feats_to_drop + lexical_features 
                    + host_feats + content_feats 
                    + url_embs_pca_10_columns + img_embs_pca_10_columns + content_embs_pca_10_columns, df_surtur)

# ### Feature Selection

# +
feat_selected_bert = feature_selection(20, scale_features(np.array(bert_url_embeddings)), 'url_bert', df_surtur)

df_surtur = pd.concat([df_surtur, feat_selected_bert], axis=1)

df_surtur['url_emb_mean_bert'] = bert_url_embs_all.mean(axis=1).values
df_surtur['url_emb_mean_red_bert'] = feat_selected_bert.mean(axis=1).values

# +
url_bert_emb_feats_bert = ['url_emb_mean_bert', 'url_emb_mean_red_bert', 'url_bert_emb_20', 
                     'url_bert_emb_21', 'url_bert_emb_22', 'url_bert_emb_23', 'url_bert_emb_24', 
                     'url_bert_emb_25', 'url_bert_emb_26', 'url_bert_emb_27', 'url_bert_emb_28', 
                     'url_bert_emb_29', 'url_bert_emb_30', 'url_bert_emb_31', 'url_bert_emb_32', 
                     'url_bert_emb_33', 'url_bert_emb_34', 'url_bert_emb_35', 'url_bert_emb_36', 
                     'url_bert_emb_37', 'url_bert_emb_38', 'url_bert_emb_39']


make_classification(url_bert_emb_feats_bert + feats_to_drop, df_surtur)
make_classification(feats_to_drop + lexical_features + host_feats
                    + content_feats + url_bert_emb_feats_bert + url_embs_pca_10_columns 
                    + img_embs_pca_10_columns + content_embs_pca_10_columns, df_surtur)
# -

# ## Best Score using feature selection

make_classification(feats_to_drop + lexical_features + host_feats
                    + content_feats + ['img_mean', 'img_mean_red', 'image_mobnet_9']
                    + ['url_emb_mean', 'url_emb_mean_red', 'url_emb_1', 
                       'url_emb_mean_bert', 'url_emb_mean_red_bert', 'url_bert_emb_20'], df_surtur)

# ## Best Score with PCA

make_classification(bert_url_embs_pca_10_columns + feats_to_drop + lexical_features 
                    + host_feats + content_feats 
                    + url_embs_pca_10_columns + img_embs_pca_10_columns + content_embs_pca_10_columns, df_surtur)

# ## Checkpoint

# +
# df_surtur.to_csv('ML_table_dataset.csv', index=False)
# -

df_surtur = pd.read_csv('ML_table_dataset.csv')


# ## More models 

def split_data(features_to_keep):
    X = df_surtur[features_to_keep]
    X = X.drop(columns = ['label', 'content', 'hostname', 'url', 'js', 
                          'domain', 'google_is_safe', 'ip_address'])
    y = df_surtur['label']

    return train_test_split(X, y, test_size=0.10, random_state=20)


# +
X_train_pca, X_test_pca, y_train_pca, y_test_pca = split_data(bert_url_embs_pca_10_columns + 
                                                              feats_to_drop + 
                                                              lexical_features + 
                                                              host_feats + 
                                                              content_feats + 
                                                              url_embs_pca_10_columns + 
                                                              img_embs_pca_10_columns + 
                                                              content_embs_pca_10_columns)

X_train_feat_sel, X_test_feat_sel, y_train_feat_sel, y_test_feat_sel = split_data(feats_to_drop + lexical_features + host_feats
                    + content_feats + ['img_mean', 'img_mean_red', 'image_mobnet_9']
                    + ['url_emb_mean', 'url_emb_mean_red', 'url_emb_1', 
                       'url_emb_mean_bert', 'url_emb_mean_red_bert', 'url_bert_emb_20'])
# -

# ### Naive Bayes 

# #### PCA mix

# +
gnb = GaussianNB()
gnb.fit(X_train_pca, y_train_pca)

y_pred = gnb.predict(X_test_pca)
print(f"Accuracy score = {accuracy_score(y_test_pca, y_pred)}")

print('\n')

print(confusion_matrix(y_test_pca, y_pred))

print('\n')

print(classification_report(y_test_pca, y_pred, target_names=['benign', 'malicious'], digits=4))
# -

# #### Feature Selection mix

# +
gnb = GaussianNB()
gnb.fit(X_train_feat_sel, y_train_feat_sel)

y_pred = gnb.predict(X_test_feat_sel)
print(f"Accuracy score = {accuracy_score(y_test_feat_sel, y_pred)}")

print('\n')

print(confusion_matrix(y_test_feat_sel, y_pred))

print('\n')

print(classification_report(y_test_feat_sel, y_pred, target_names=['benign', 'malicious'], digits=4))
# -

# ### SVM 

# #### PCA mix

# +
svm_model = svm.SVC(C=1000, gamma=0.001, verbose=3)
svm_model.fit(X_train_pca, y_train_pca)

y_pred = svm_model.predict(X_test_pca)
print(f"Accuracy score = {accuracy_score(y_test_pca, y_pred)}")

print('\n')

print(confusion_matrix(y_test_pca, y_pred))

print('\n')

print(classification_report(y_test_pca, y_pred, target_names=['benign', 'malicious'], digits=4))
# -

# #### Feature Selection mix

# +
svm_model = svm.SVC(C=1000, gamma=0.001, verbose=3)
svm_model.fit(X_train_feat_sel, y_train_feat_sel)

y_pred = svm_model.predict(X_test_feat_sel)
print(f"Accuracy score = {accuracy_score(y_test_feat_sel, y_pred)}")

print('\n')

print(confusion_matrix(y_test_feat_sel, y_pred))

print('\n')

print(classification_report(y_test_feat_sel, y_pred, target_names=['benign', 'malicious'], digits=4))
# -

# ### KNN

# #### PCA mix

# +
knn_model = KNeighborsClassifier(n_neighbors=100)
knn_model.fit(X_train_pca, y_train_pca)

y_pred = knn_model.predict(X_test_pca)
print(f"Accuracy score = {accuracy_score(y_test_pca, y_pred)}")

print('\n')

print(confusion_matrix(y_test_pca, y_pred))

print('\n')

print(classification_report(y_test_pca, y_pred, target_names=['benign', 'malicious'], digits=4))
# -

# #### Feature Selection mix

# +
knn_model = KNeighborsClassifier(n_neighbors=100)
knn_model.fit(X_train_feat_sel, y_train_feat_sel)

y_pred = knn_model.predict(X_test_feat_sel)
print(f"Accuracy score = {accuracy_score(y_test_feat_sel, y_pred)}")

print('\n')

print(confusion_matrix(y_test_feat_sel, y_pred))

print('\n')

print(classification_report(y_test_feat_sel, y_pred, target_names=['benign', 'malicious'], digits=4))
# -

# ### Adaboost

# #### PCA mix

# +
adaboost_model = AdaBoostClassifier()
adaboost_model.fit(X_train_pca, y_train_pca)

y_pred = adaboost_model.predict(X_test_pca)
print(f"Accuracy score = {accuracy_score(y_test_pca, y_pred)}")

print('\n')

print(confusion_matrix(y_test_pca, y_pred))

print('\n')

print(classification_report(y_test_pca, y_pred, target_names=['benign', 'malicious'], digits=4))
# -

# #### Feature Selection mix

# +
adaboost_model = AdaBoostClassifier()
adaboost_model.fit(X_train_feat_sel, y_train_feat_sel)

y_pred = adaboost_model.predict(X_test_feat_sel)
print(f"Accuracy score = {accuracy_score(y_test_feat_sel, y_pred)}")

print('\n')

print(confusion_matrix(y_test_feat_sel, y_pred))

print('\n')

print(classification_report(y_test_feat_sel, y_pred, target_names=['benign', 'malicious'], digits=4))
# -

# ## Random Forest 

# +
from sklearn.ensemble import RandomForestClassifier

clf = RandomForestClassifier(max_depth=16, random_state=0)
clf.fit(X_train_pca, y_train_pca)

y_pred = clf.predict(X_test_pca)

print(f"Accuracy score = {accuracy_score(y_test_pca, y_pred)}")

print('\n')

print(confusion_matrix(y_test_pca, y_pred))

print('\n')

print(classification_report(y_test_pca, y_pred, target_names=['benign', 'malicious'], digits=4))

# +
from sklearn.ensemble import RandomForestClassifier

clf = RandomForestClassifier(max_depth=16, random_state=0)
clf.fit(X_train_feat_sel, y_train_feat_sel)

y_pred = clf.predict(X_test_feat_sel)

print(f"Accuracy score = {accuracy_score(y_test_feat_sel, y_pred)}")

print('\n')

print(confusion_matrix(y_test_feat_sel, y_pred))

print('\n')

print(classification_report(y_test_feat_sel, y_pred, target_names=['benign', 'malicious'], digits=4))

StarcoderdataJupyterScriptsDedupFiltered