<jupyter_start><jupyter_text>Network Intrusion Detection<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
import keras
from keras.callbacks import ModelCheckpoint
from keras.callbacks import EarlyStopping
from keras.models import Sequential
from keras.layers.core import Dense, Activation, Flatten
from keras.optimizers import Adam
import pandas as pd
import io
import requests
import numpy as np
from sklearn import metrics
import os
import json
import csv
import pandas as pd
import sklearn.feature_extraction.text as tfidf
from sklearn.model_selection import train_test_split
from sklearn import datasets,linear_model, preprocessing,utils
from sklearn.metrics import mean_squared_error,r2_score
from scipy.stats import zscore
import numpy as np
import pickle
from sklearn.preprocessing import LabelEncoder
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import MultinomialNB
from sklearn import svm
from sklearn.metrics import f1_score
import collections
from keras import optimizers
from keras.layers import Conv1D, Conv2D, MaxPooling2D

import collections
from sklearn import preprocessing
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import shutil
import os


%matplotlib inline
import matplotlib.pyplot as plt
from sklearn.metrics import roc_curve, auc

from sklearn.model_selection import train_test_split
import tensorflow as tf
import numpy as np
from sklearn import metrics
from keras.models import Sequential
from keras.layers.core import Dense, Activation
from keras.callbacks import EarlyStopping
from keras.callbacks import ModelCheckpoint


from sklearn import svm, datasets
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, classification_report<jupyter_output><empty_output><jupyter_text><jupyter_code># Plot a confusion matrix.
# cm is the confusion matrix, names are the names of the classes.
def plot_confusion_matrix(cm, names, title='Confusion matrix', cmap=plt.cm.Blues):
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(names))
    plt.xticks(tick_marks, names, rotation=45)
    plt.yticks(tick_marks, names)
    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')
    

# Plot an ROC. pred - the predictions, y - the expected output.
def plot_roc(pred,y):
    fpr, tpr, thresholds = roc_curve(y, pred)
    roc_auc = auc(fpr, tpr)

    plt.figure()
    plt.plot(fpr, tpr, label='ROC curve (area = %0.2f)' % roc_auc)
    plt.plot([0, 1], [0, 1], 'k--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('Receiver Operating Characteristic (ROC)')
    plt.legend(loc="lower right")
    plt.show()<jupyter_output><empty_output><jupyter_text>Data Preprocessing   Step 1: Read network intrusion data csv file<jupyter_code>df = pd.read_csv("/content/drive/My Drive/Colab Notebooks/Data_set_KDD_cup/kddcup99_csv.csv")
df.head()<jupyter_output><empty_output><jupyter_text>Step 2 Adding Column Headers<jupyter_code>df.columns = [    'duration',    'protocol_type',    'service',     'flag',    'src_bytes',    'dst_bytes',    'land',    'wrong_fragment',
    'urgent',    'hot',    'num_failed_logins',    'logged_in',    'num_compromised',    'root_shell',    'su_attempted',    'num_root',    'num_file_creations',    'num_shells',    'num_access_files',    'num_outbound_cmds',     'is_host_login',    'is_guest_login',    'count',    'srv_count',    'serror_rate',    'srv_serror_rate',    'rerror_rate',    'srv_rerror_rate',    'same_srv_rate',    'diff_srv_rate',    'srv_diff_host_rate',    'dst_host_count',    'dst_host_srv_count',    'dst_host_same_srv_rate',    'dst_host_diff_srv_rate',    'dst_host_same_src_port_rate',    'dst_host_srv_diff_host_rate',    'dst_host_serror_rate',    'dst_host_srv_serror_rate',    'dst_host_rerror_rate',    'dst_host_srv_rerror_rate',    'outcome' ] 
df.head()<jupyter_output><empty_output><jupyter_text>Step 3 Encoding good connections as “0” and bad connections as “1”<jupyter_code>df['outcome'][:1]
df.groupby('outcome').count()
df.groupby('outcome').head()
def classify_connections(outcome):
    if(outcome =="normal"):
        outcome = 0  
    else: 
        outcome = 1
    
    return outcome
df["outcome"] = df["outcome"].apply(classify_connections)
df["outcome"][9020]
df.count()[1]<jupyter_output><empty_output><jupyter_text>Step 4 Drop duplicates<jupyter_code>df = df.drop_duplicates()
df.count()[0]<jupyter_output><empty_output><jupyter_text>Step 5 Drop the rows where at least one element is missing<jupyter_code>df = df.dropna()
df.count()[0]     # shows that there is no row with atleast one element missing<jupyter_output><empty_output><jupyter_text>Data Preprocessing Completed<jupyter_code>df.values[:5]
df["wrong_fragment"].values[:1000]<jupyter_output><empty_output><jupyter_text>df.columns = [    'duration',    'protocol_type',    'service',     'flag',    'src_bytes',    'dst_bytes',    'land',    'wrong_fragment', 'urgent',    'hot',    'num_failed_logins',    'logged_in',    'num_compromised',    'root_shell',    'su_attempted',    'num_root',    'num_file_creations',    'num_shells',    'num_access_files',    'num_outbound_cmds',     'is_host_login',    'is_guest_login',    'count',    'srv_count',    'serror_rate',    'srv_serror_rate',    'rerror_rate',    'srv_rerror_rate',    'same_srv_rate',    'diff_srv_rate',    'srv_diff_host_rate',    'dst_host_count',    'dst_host_srv_count',    'dst_host_same_srv_rate',    'dst_host_diff_srv_rate',    'dst_host_same_src_port_rate',    'dst_host_srv_diff_host_rate',    'dst_host_serror_rate',    'dst_host_srv_serror_rate',    'dst_host_rerror_rate',    'dst_host_srv_rerror_rate',    'outcome' ]<jupyter_code>df.groupby('outcome').count()
df.groupby('outcome').head()
def Z_score(column):
    df[column] = zscore(df[column])
mylist = ['duration','src_bytes','dst_bytes','hot','num_failed_logins','num_compromised','num_root','num_file_creations','num_access_files','count','srv_count', 'dst_host_count', 'dst_host_srv_count']
for i in mylist:
    Z_score(i)
df.head()
df
# Encode text values to dummy variables(i.e. [1,0,0],[0,1,0],[0,0,1] for red,green,blue)
def encode_text_dummy(df, name):
    dummies = pd.get_dummies(df[name])
    for x in dummies.columns:
        dummy_name = "{}-{}".format(name, x)
        df[dummy_name] = dummies[x]
    df.drop(name, axis=1, inplace=True)
mylist = ["protocol_type","service","flag","logged_in","root_shell","is_guest_login"]
for i in mylist:
    print(i)
    encode_text_dummy(df,i)
df
df.drop(['num_outbound_cmds', 'is_host_login'], axis=1)<jupyter_output><empty_output><jupyter_text>LOGISTIC REGRESSION<jupyter_code>output_df = df["outcome"]
output_array = output_df.values

print(output_array[:1])
df = df.drop(["outcome"], axis=1)
input_df = df
input_array = input_df.values
x_train,x_test,y_train,y_test = train_test_split(input_array,output_array, test_size=0.2,random_state=42)
print(x_train.shape)
logreg = linear_model.LogisticRegression()
print(type(y_train))
print(y_train)
print(utils.multiclass.type_of_target(y_train))
logreg.fit(x_train, y_train)
y_pred_logistic = logreg.predict(x_test)
# Explained f1 score: 1 is perfect prediction
print('Evaluating with F1 score metric: %.2f' % f1_score(y_test, y_pred_logistic,average='micro'))
score = metrics.accuracy_score(y_test, y_pred_logistic)
print("Accuracy score: {}".format(score))
score = metrics.precision_score(y_test,y_pred_logistic, average= "weighted")
print("Precision score: {}".format(score))
score = metrics.recall_score(y_test,y_pred_logistic, average= "weighted")
print("Recall score: {}".format(score))
names = (np.unique(output_array))<jupyter_output><empty_output><jupyter_text>Confusion Matrix<jupyter_code># Compute confusion matrix
cm = confusion_matrix(y_test, y_pred_logistic)
print(cm)

print('Plotting confusion matrix')
plt.figure()
plot_confusion_matrix(cm, names)
plt.show()

print(classification_report(y_test, y_pred_logistic))<jupyter_output>[[17511    93]
 [  123 11390]]
Plotting confusion matrix<jupyter_text>ROC Curve<jupyter_code>plot_roc(y_pred_logistic,y_test)<jupyter_output><empty_output><jupyter_text>Nearest Neighbor<jupyter_code>neigh = KNeighborsClassifier(n_neighbors=23)
neigh.fit(x_train, y_train)
y_pred_neigh = neigh.predict(x_test)
# Explained variance score: 1 is perfect prediction
print('Evaluating with F1 score metric: %.2f' % f1_score(y_test, y_pred_neigh))
score = metrics.accuracy_score(y_test, y_pred_neigh)
print("Accuracy score: {}".format(score))
score = metrics.precision_score(y_test,y_pred_neigh, average= "weighted")
print("Precision score: {}".format(score))
score = metrics.recall_score(y_test,y_pred_neigh, average= "weighted")
print("Recall score: {}".format(score))
# Compute confusion matrix
cm = confusion_matrix(y_test,y_pred_neigh)
print(cm)

print('Plotting confusion matrix')

plt.figure()
plot_confusion_matrix(cm, names)
plt.show()

print(classification_report(y_test, y_pred_neigh))
plot_roc(y_pred_neigh,y_test)<jupyter_output><empty_output><jupyter_text>MULTINOMIAL NAIVE BAYES<jupyter_code>from sklearn.preprocessing import MinMaxScaler
 scaler = MinMaxScaler()
input_array_min_max = scaler.fit_transform(input_array)
x_train,x_test,y_train,y_test = train_test_split(input_array_min_max,output_array, test_size=0.2,random_state=42)
clf = MultinomialNB()
clf.fit(x_train, y_train)
clf_predict = clf.predict(x_test)
# Explained variance score: 1 is perfect prediction
print('Evaluating with F1 score metric: %.2f' % f1_score(y_test, clf_predict))
score = metrics.accuracy_score(y_test, clf_predict)
print("Accuracy score: {}".format(score))
score = metrics.precision_score(y_test,clf_predict, average= "weighted")
print("Precision score: {}".format(score))
score = metrics.recall_score(y_test,clf_predict, average= "weighted")
print("Recall score: {}".format(score))
# Compute confusion matrix
cm = confusion_matrix(y_test, clf_predict)
print(cm)

print('Plotting confusion matrix')

plt.figure()
plot_confusion_matrix(cm, names)
plt.show()

print(classification_report(y_test, clf_predict))
plot_roc(clf_predict,y_test)<jupyter_output><empty_output><jupyter_text>SUPPORT VECTOR MACHINE<jupyter_code>SVM_classifier =  svm.SVC()
SVM_classifier.fit(x_train, y_train)
y_pred_SVM = SVM_classifier.predict(x_test)
# Explained variance score: 1 is perfect prediction
print('Evaluating with F1 score metric: %.2f' % f1_score(y_test, y_pred_SVM, average='micro'))
score = metrics.accuracy_score(y_test, y_pred_SVM)
print("Accuracy score: {}".format(score))
score = metrics.precision_score(y_test,y_pred_SVM, average= "weighted")
print("Precision score: {}".format(score))
score = metrics.recall_score(y_test,y_pred_SVM, average= "weighted")
print("Recall score: {}".format(score))
# Compute confusion matrix
cm = confusion_matrix(y_test, y_pred_SVM)
print(cm)

print('Plotting confusion matrix')

plt.figure()
plot_confusion_matrix(cm, names)
plt.show()

print(classification_report(y_test, y_pred_SVM))
plot_roc(y_pred_SVM,y_test)<jupyter_output><empty_output><jupyter_text>Gaussian Naive Bayes<jupyter_code>from sklearn.naive_bayes import GaussianNB
GaussianNB_clf = GaussianNB()
GaussianNB_clf.fit(x_train, y_train)
print(x_train.shape)
y_test1 = y_test.reshape((y_test.shape[0], 1))
GaussianNB_pred = GaussianNB_clf.predict(x_test)
# Explained variance score: 1 is perfect prediction
print('Evaluating with F1 score metric: %.2f' % f1_score(y_test1, GaussianNB_pred, average='micro'))
score = metrics.accuracy_score(y_test1, GaussianNB_pred)
print("Accuracy score: {}".format(score))
score = metrics.precision_score(y_test1,GaussianNB_pred, average= "weighted")
print("Precision score: {}".format(score))
score = metrics.recall_score(y_test1,GaussianNB_pred, average= "weighted")
print("Recall score: {}".format(score))
# Compute confusion matrix
cm = confusion_matrix(y_test1, GaussianNB_pred)
print(cm)

print('Plotting confusion matrix')

plt.figure()
plot_confusion_matrix(cm, names)
plt.show()

print(classification_report(y_test1, GaussianNB_pred))
plot_roc(GaussianNB_pred,y_test1)
print(x_train.shape)
print(y_train.shape)
num_classes = 2
#Convert class vectors to one hot format
y_train = keras.utils.to_categorical(y_train,num_classes)
print(y_train[:1])
y_test = keras.utils.to_categorical(y_test,num_classes)
print(x_train.shape)
print(y_train.shape)<jupyter_output>(116465, 121)
(116465, 2)<jupyter_text>Tensorflow Classification<jupyter_code>for i in range(5):
    model_class = Sequential()
    model_class.add(Dense(50, input_dim=x_train.shape[1], activation='relu')) # Hidden 1
    model_class.add(Dense(25, activation='relu')) # Hidden 2
    model_class.add(Dense(y_train.shape[1], activation='softmax')) # Output
    #model_class.compile(optimizer = 'adam' , loss = 'categorical_crossentropy', metrics = ['accuracy'])
    model_class.compile(optimizer = 'adam' , loss = 'categorical_crossentropy', metrics = ['accuracy'])
    monitor = EarlyStopping(monitor='val_loss', min_delta=1e-3, patience=2, verbose=2, mode='auto') 
    checkpointer = ModelCheckpoint(filepath="best_weights_class.hdf5", verbose=0, save_best_only=True) # save best model
    model_class.fit(x_train, y_train,validation_data=(x_test,y_test),callbacks=[monitor,checkpointer],verbose=2,epochs=10)
model_class.load_weights('best_weights_class.hdf5') # load weights from best model
Tf_class_pred = model_class.predict(x_test)
pred1 = np.argmax(Tf_class_pred ,axis=1) # raw probabilities to chosen class (highest probability)
print(pred1)
y_true= np.argmax(y_test,axis=1) 
score = metrics.accuracy_score(y_true, pred1)
print("Accuracy score: {}".format(score))
# Explained variance score: 1 is perfect prediction
print('Evaluating with F1 score metric: %.2f' % f1_score(y_true,pred1 , average='micro'))
pre_score = metrics.precision_score(y_true, pred1, average= "weighted")
print("Precision score: {}".format(pre_score))
score = metrics.recall_score(y_true,pred1, average= "weighted")
print("Recall score: {}".format(score))
# Compute confusion matrix
cm = confusion_matrix(y_true,pred1)
print(cm)

print('Plotting confusion matrix')

plt.figure()
plot_confusion_matrix(cm, names)
plt.show()

print(classification_report(y_true, pred1))
plot_roc(y_true,pred1)<jupyter_output><empty_output><jupyter_text>CNN<jupyter_code>print(x_train.shape)
print(y_train.shape)
x_train2 = x_train.reshape((116465,1,121,1))

print(x_train2[:2])
print(x_train2.shape)
# define a CNN
cnn = Sequential()
cnn.add(Conv2D(64, kernel_size=(1, 10), strides=(1, 1),padding='valid',
                 activation='relu',
                 input_shape=(1,121,1)))

    # the above code is equivalent to 
    # model.add(Conv1D(64, kernel_size=3, strides=1, activation='relu', input_shape=(128, 1)))

cnn.add(MaxPooling2D(pool_size=(1,2)))

cnn.add(Flatten())
cnn.add(Dense(30, activation="relu"))

cnn.summary()   
cnn.add(Dense(num_classes, activation="softmax"))
cnn.summary()
    
# define optimizer and objective, compile cnn
cnn.compile(loss="categorical_crossentropy", optimizer="adam" , metrics = ['accuracy'])
monitor = EarlyStopping(monitor='val_loss', min_delta=1e-3, patience=2, verbose=2, mode='auto') 
checkpointer = ModelCheckpoint(filepath="best_weights_cnn.hdf5", verbose=0, save_best_only=True) # save best model
cnn.fit(x_train2, y_train,validation_split=0.2,callbacks=[monitor,checkpointer],verbose=2,epochs=10)
cnn.load_weights('best_weights_cnn.hdf5') # load weights from best model
x_test = x_test.reshape((-1,1,121,1))
print(x_test.shape)
pred = cnn.predict(x_test)
#pred = np.argmax(pred,axis=1)
print("Shape: {}".format(pred.shape))
print(pred)
print(y_test.shape)
pred=pred.reshape((-1,2))
print(pred.shape)
score = metrics.accuracy_score(y_test, pred.round())
print('Accuracy: {}'.format(score))
f1 = metrics.f1_score(y_test, pred.round(), average='weighted')
print('Averaged F1: {}'.format(f1))
print(metrics.classification_report(y_test, pred.round()))
score = metrics.precision_score(y_test,pred.round(), average= "weighted")
print("Precision score: {}".format(score))
score = metrics.recall_score(y_test,pred.round(), average= "weighted")
print("Recall score: {}".format(score))
plot_roc(pred.round().argmax(axis=1),y_test.argmax(axis=1))
# Compute confusion matrix
cm = confusion_matrix(y_test.argmax(axis=1), pred.round().argmax(axis=1))
print(cm)

print('Plotting confusion matrix')

plt.figure()
plot_confusion_matrix(cm, names)
plt.show()

print(classification_report(y_test.argmax(axis=1), pred.round().argmax(axis=1)))<jupyter_output>[[17583    21]
 [   40 11473]]
Plotting confusion matrix
StarcoderdataJupyterStructuredCleanDedup