<jupyter_start><jupyter_text>Read data<jupyter_code>training_df = cudf.read_csv('../data/features/UNSW_NB15_training-set.csv')
testing_df  = cudf.read_csv('../data/features/UNSW_NB15_testing-set.csv')

### ADD IDENTIFIER
training_df['test'] = 0
testing_df['test']  = 1

df = cudf.concat([training_df, testing_df])
df = df.drop(['id']).reset_index().rename({'index': 'id'})

df.head()
to_keep = [
      'dur', 'proto', 'service', 'state', 'spkts', 'dpkts', 'sbytes', 'dbytes'
    , 'sttl', 'dttl', 'sload', 'dload', 'sloss', 'dloss'
    , 'sjit', 'djit', 'swin', 'stcpb', 'dtcpb', 'dwin', 'tcprtt', 'synack', 'ackdat'
    , 'ct_srv_src', 'ct_state_ttl', 'ct_src_dport_ltm', 'ct_dst_sport_ltm', 'ct_dst_src_ltm'
    , 'ct_srv_dst', 'attack_cat', 'label', 'test']

df = df[to_keep].reset_index(drop=True).reset_index()
df = df.rename({'index': 'id'})
df['id'] = df['id'].astype('int32')
df['attack_cat'] = df['attack_cat'].fillna('Normal')
df['attack_cat'] = df['attack_cat'].str.replace(' ', '')
df.describe()<jupyter_output><empty_output><jupyter_text>Encode variables  Continuous to bins<jupyter_code>bin_num = 10

quantiles = cudf.DataFrame()

for i in range(1, bin_num):
    quant = i / 10
    print(f'Quantile: {quant}')
    
    quantiles['q' + str(i)] = df.quantile(q=quant)

quantiles = quantiles.dropna().reset_index()
quantiles_list = quantiles.to_pandas().to_dict('records')
quantiles = []

for q in quantiles_list:
    if q['index'] not in ['id', 'label', 'test']:
        elements = list(q.items())
        quants = sorted(list(set([0.0] + [e[1] for e in elements[1:]])))
        quantiles.append((elements[0][1], quants))
def encode_quantiles(df, quantiles):
    temp_df = cudf.DataFrame()
    
    for q in quantiles:
        col_name = q[0]
        bins = q[1]

        if df[col_name].dtype == 'int64':
            bins = [int(e) for e in bins]

        temp_df[col_name + '_bin'] = df[col_name].digitize(np.array(bins))
    return temp_df

carry_over_cols = ['proto', 'service', 'state', 'attack_cat', 'label', 'test']
df_binned = encode_quantiles(df, quantiles)

for col in carry_over_cols:
    df_binned[col] = df[col]

df_binned = df_binned.reset_index().rename({'index': 'id'})
# del df
df_binned.head()<jupyter_output><empty_output><jupyter_text>Categorical to index<jupyter_code>import cuml

encoders = {}
cols_to_encode = ['proto', 'service', 'state']

for col in cols_to_encode:
    le = cuml.preprocessing.LabelEncoder()
    df_binned[col] = df_binned[col].astype('category')
    df_binned[col + '_enc'] = le.fit_transform(df_binned[col])
    
    encoders[col] = le
    
cols_reordered = [c for c in df_binned.columns if c not in cols_to_encode + ['label']] + ['label']
df_binned = df_binned[cols_reordered]
df_binned.columns<jupyter_output><empty_output><jupyter_text>Explode to COO format<jupyter_code>### Attack_cat encoding
attack_cat = [
      (0, 'Normal')
    , (1, 'Reconnaissance')
    , (2, 'Shellcode')
    , (3, 'Analysis')
    , (4, 'Backdoor')
    , (5, 'DoS')
    , (6, 'Exploits')
    , (7, 'Generic')
    , (8, 'Fuzzers')
    , (9, 'Worms')
]

attack_cat = {
      'attack_id':  [e[0] for e in attack_cat]
    , 'attack_cat': [e[1] for e in attack_cat]
}

attack_categories = cudf.DataFrame(attack_cat)
df_binned = df_binned.merge(attack_categories, on='attack_cat')
df_binned = df_binned.drop('attack_cat')
df_binned.head()
training_df = df_binned.query('test == 0')
training_df = training_df.drop(['test', 'id']).reset_index().rename({'index': 'id'})

# training_df.head()
df_exploded = cudf.melt(training_df, id_vars=['id', 'label', 'attack_id']).sort_values('id').reset_index(drop=True)
df_exploded['variable'] = df_exploded['variable'].astype('str')
df_exploded['value'] = df_exploded['value'].astype('str')
df_exploded['feature'] = df_exploded['variable'] + '=' + df_exploded['value']
df_exploded.drop(['variable', 'value'])

feature_encoding = df_exploded['feature'].unique().reset_index()
len(df_binned), len(training_df)
feature_encoding['index'] = feature_encoding['index'].astype('int16')
feature_encoding = feature_encoding.rename({'index': 'feature_enc'})
feature_encoding.head()
df_exploded = df_exploded.merge(feature_encoding, on='feature')[['id', 'feature_enc', 'label', 'attack_id']]
df_exploded.head()<jupyter_output><empty_output><jupyter_text>Finding frequent patterns<jupyter_code>def mine_patterns(df_coo, df_binned, min_attack_rate=0.75, min_feature_count=200, max_iter=-1):
    def return_rank(frate, counts, rank):
        for i, (f, c) in enumerate(zip(frate, counts)):
            rank[i] = math.log(float(c)) * f
            
    features = df_coo['feature_enc'].unique().to_frame()
    df_coo = df_coo.merge(features, on='feature_enc')

    #### FIND FREQUENT ITEMS
    freq_items = df_coo.groupby(['feature_enc']).agg({'id': 'count', 'label': 'sum'}).reset_index()
    freq_items['attack_rate'] = freq_items['label'] / freq_items['id']

    freq_items = freq_items.apply_rows(
        return_rank
        , incols = {'label': 'counts', 'attack_rate': 'frate'}
        , outcols = {'rank': np.float64}
        , kwargs = {}
    ) 

    freq_items = freq_items.sort_values('rank', ascending=False)
    freq_items.head(5)

    freq_items = freq_items.query(f'attack_rate >= {min_attack_rate} and label > {min_feature_count}')

    if max_iter == -1:
        max_iter = len(freq_items)

    devices_checked = {}

    patterns = []
    stats    = []

    features_ordered = list(freq_items['feature_enc'].to_array())

    for i in range(max_iter):
        feature = features_ordered[i]

        ### get all the ids
        ids = df_coo.query('feature_enc == @feature')['id'].unique().to_frame()
        h = ids.hash_columns(['id']).sum()

        if h not in devices_checked:    
            count_ids = len(ids)
            devices_checked[h] = 1

            ### OUTPUT PATTERN
            all_features = df_coo.merge(ids, on='id').groupby('feature_enc').agg({'label': 'count'}).query('label == @count_ids').reset_index()
            all_features['pattern_id'] = i

            patterns.append(all_features[['pattern_id', 'feature_enc']])

            ### OUTPUT STATS
            ids = ids.merge(df_binned, on='id')
            ids['pattern_id'] = i
            ids = ids.groupby('pattern_id').agg({'id': 'count', 'label': 'sum'})
            ids = ids.rename({'id': 'packet_count', 'label': 'attack_count'})
            ids['attack_rate'] = ids['attack_count'] / ids['packet_count']
            ids['feature_cnt'] = all_features['feature_enc'].count()
            ids = ids.reset_index()
            stats.append(ids)
            
    patterns = cudf.concat(patterns).merge(features, on='feature_enc').sort_values(by='pattern_id')
    stats = cudf.concat(stats)
    
    return patterns, stats<jupyter_output>time: 10.6 ms<jupyter_text>Testing<jupyter_code>testing_binned = df_binned.query('test == 1')<jupyter_output>time: 233 ms<jupyter_text>Pattern encoding<jupyter_code>def encode_patterns(binned_df, patterns_to_encode):
    subset = binned_df[['id', 'label']]
    subset['pred'] = 0

    for i, pattern in enumerate(patterns_to_encode):
        q = ' and '.join([' == '.join(e) for e in pattern])
        temp = binned_df.query(q)['id'].to_frame()
        temp['enc'] = 1

        subset = subset.merge(temp, on=['id'], how='left')
        subset = subset.rename({'enc': 'col_' + str(i)})
        subset['col_' + str(i)] = subset['col_' + str(i)].fillna(0)
        subset['col_' + str(i)] = subset['col_' + str(i)].astype('float32')
        subset['pred'] = subset['pred'] + subset['col_'+str(i)]
        

    subset['pred'] = subset['pred'] > 0
    subset['pred'] = subset['pred'].astype('int8')
    
    return subset<jupyter_output>time: 1.86 ms<jupyter_text>End-2-end pattern mining<jupyter_code>def calculate_metrics(df):
    ttl = df['id'].sum()
    accuracy = df.query('(label == 0 and pred == 0) or (label == 1 and pred == 1)')['id'].sum() / ttl
    
    fp = df.query('(label == 0 and pred == 1)')['id'].sum()
    tn = df.query('(label == 0 and pred == 0)')['id'].sum()
    fn = df.query('(label == 1 and pred == 0)')['id'].sum()
    tp = df.query('(label == 1 and pred == 1)')['id'].sum()
    
    fpr = fp / (fp + tn)
    fnr = fn / (fn + tp)
    
    return fp, tn, fn, tp, accuracy, fpr, fnr, (fpr + fnr) / 2

def encode_patterns(binned_df, patterns_to_encode):
    subset = binned_df[['id', 'label']]
    subset['pred'] = 0

    for i, pattern in enumerate(patterns_to_encode):
        q = ' and '.join([' == '.join(e) for e in pattern])
        temp = binned_df.query(q)['id'].to_frame()
        temp['enc'] = 1

        subset = subset.merge(temp, on=['id'], how='left')
        subset = subset.rename({'enc': 'col_' + str(i)})
        subset['col_' + str(i)] = subset['col_' + str(i)].fillna(0)
        subset['col_' + str(i)] = subset['col_' + str(i)].astype('float32')
        subset['pred'] = subset['pred'] + subset['col_'+str(i)]
        

    subset['pred'] = subset['pred'] > 0
    subset['pred'] = subset['pred'].astype('int8')
    
    return subset

def e2e_pattern_mining(label, df, training_df, testing_df, feature_encoding, results, min_attack_rate=.5, min_feature_count=200):
    print('[{0}] Mining features...'.format(label))
    patterns, stats = mine_patterns(df, training_df, min_attack_rate=min_attack_rate, min_feature_count=min_feature_count)
    patterns['label'] = label
    stats['label'] = label
    
    print('[{0}] Encoding patterns...'.format(label))
    patterns_rec = patterns.merge(feature_encoding, on='feature_enc')
    
    patterns_rec['col_name'] = patterns_rec['feature'].str.split('=')[0]
    patterns_rec['col_val']  = patterns_rec['feature'].str.split('=')[1]
    patterns_rec = patterns_rec.sort_values(by='pattern_id').reset_index(drop=True)
    patterns_expl = patterns_rec[['pattern_id', 'col_name', 'col_val']].to_pandas().to_records()

    patterns_to_encode = []

    curr = 0
    temp = []
    for i in patterns_expl:
        if curr == i[1]:
            temp.append((i[2], i[3]))

        else:
            patterns_to_encode.append(temp)
            temp = [(i[2], i[3])]
        curr = i[1]

    encoded_testing = encode_patterns(testing_df, patterns_to_encode)
    encoded_testing['id'] = encoded_testing['id'].astype('float32')

    #### Encoding training dataset
    encoded_df = encode_patterns(training_df, patterns_to_encode)
    X = encoded_df[['col_' + str(i) for i in range(len(patterns_to_encode))]]
    y = encoded_df['label'].astype('int32')
        
    #### Binned data
    X_binned = training_df[[col for col in training_df.columns if col not in ['id', 'label', 'attack_id']]]

    for col in X_binned:
        X_binned[col] = X_binned[col].astype('float32')

    y_binned = training_df['label'].astype('float32')
    
    X_testing_binned = testing_df[[col for col in testing_df.columns if col not in ['id', 'test', 'label', 'attack_id']]]
    for col in X_testing_binned:
        X_testing_binned[col] = X_testing_binned[col].astype('float32')
        
    y_testing_binned = testing_df[['id', 'label']]

   
    print('[{0}] Building models'.format(label))
    #####################################
    #### SIMPLE ENCODING
    #####################################
    pred = encoded_testing.groupby(['label', 'pred']).agg({'id': 'count'}).reset_index()
    simple_results = calculate_metrics(pred)
    
    row_to_insert = {
        'label': label
        , 'model': 'simple'
        , 'fp': simple_results[0]
        , 'tn': simple_results[1]
        , 'fn': simple_results[2]
        , 'tp': simple_results[3]
        , 'accuracy': simple_results[4]
        , 'fpr': simple_results[5]
        , 'fnr': simple_results[6]
        , 'far': simple_results[7]
    }
    
    simple_results_df = cudf.DataFrame(row_to_insert)
    
    #####################################
    #### RANDOM FOREST
    #####################################
    print('[{0}]\tRandom Forest'.format(label))
    rf = cuRFC(max_features=1.0, n_estimators=100, n_bins=10)
    rf.fit(X, y)

    encoded_testing = encoded_testing.drop('pred')
    encoded_testing['pred'] = rf.predict(encoded_testing[['col_' + str(i) for i in range(len(patterns_to_encode))]])
    rf_conf = encoded_testing.groupby(['label', 'pred']).agg({'id': 'count'}).reset_index()
    rf_results = calculate_metrics(rf_conf)
    
    row_to_insert = {
        'label': label
        , 'model': 'random forest'
        , 'fp': rf_results[0]
        , 'tn': rf_results[1]
        , 'fn': rf_results[2]
        , 'tp': rf_results[3]
        , 'accuracy': rf_results[4]
        , 'fpr': rf_results[5]
        , 'fnr': rf_results[6]
        , 'far': rf_results[7]
    }

    rf_results_df = cudf.DataFrame(row_to_insert)
    
    #####################################
    #### SUPPORT VECTOR MACHINES
    #####################################
    print('[{0}]\tSupport Vector Machines'.format(label))
    svc = SVC()
    svc.fit(X, y)

    encoded_testing = encoded_testing#.drop('pred')
    encoded_testing['pred'] = svc.predict(encoded_testing[['col_' + str(i) for i in range(len(patterns_to_encode))]])
    svc_conf = encoded_testing.groupby(['label', 'pred']).agg({'id': 'count'}).reset_index()
    svc_results = calculate_metrics(svc_conf)
    
    row_to_insert = {
        'label': label
        , 'model': 'SVC'
        , 'fp': svc_results[0]
        , 'tn': svc_results[1]
        , 'fn': svc_results[2]
        , 'tp': svc_results[3]
        , 'accuracy': svc_results[4]
        , 'fpr': svc_results[5]
        , 'fnr': svc_results[6]
        , 'far': svc_results[7]
    }
    svc_results_df = cudf.DataFrame(row_to_insert)
    results = cudf.concat([simple_results_df, rf_results_df, svc_results_df])

    return patterns, stats, results
results  = cudf.DataFrame()
patterns = cudf.DataFrame()
stats    = cudf.DataFrame()

#### overall label
p, s, r = e2e_pattern_mining('0/1 Overall Label', df_exploded, training_df, testing_binned, feature_encoding, results, min_attack_rate=.85)
results  = cudf.concat([results,  r])
patterns = cudf.concat([patterns, p])
stats    = cudf.concat([stats,    s])

results
def redefine_label(attack_id, new_label, attack_select):
    for i, ai in enumerate(attack_id):
        new_label[i] = 1 if ai == attack_select else 0
        

for i, ac in list(zip(attack_cat['attack_id'], attack_cat['attack_cat']))[1:]:     
    training_df_new = training_df.apply_rows(
        redefine_label
        , incols  = ['attack_id']
        , outcols = {'new_label': np.int32}
        , kwargs  = {'attack_select': i}
    )

    testing_binned_new = testing_binned.apply_rows(
        redefine_label
        , incols  = ['attack_id']
        , outcols = {'new_label': np.int32}
        , kwargs  = {'attack_select': i}
    )

    training_df_new = training_df_new.drop('label').rename({'new_label': 'label'})
    testing_binned_new = testing_binned_new.drop('label').rename({'new_label': 'label'})

    p, s, r = e2e_pattern_mining(ac + ' (full)', df_exploded, training_df_new, testing_binned_new, feature_encoding, results, min_attack_rate=.85)
    
    results  = cudf.concat([results,  r])
    patterns = cudf.concat([patterns, p])
    stats    = cudf.concat([stats,    s])
    
# results
for i, ac in list(zip(attack_cat['attack_id'], attack_cat['attack_cat']))[1:]:     
#     print(i, ac)
    training_df_new = training_df.query('attack_id == 0 or attack_id == @i')
    testing_binned_new = testing_binned.query('attack_id == 0 or attack_id == @i')

    p, s, r = e2e_pattern_mining(ac + ' (limited)', df_exploded, training_df_new, testing_binned_new, feature_encoding, results, min_attack_rate=.85)
    
    results  = cudf.concat([results,  r])
    patterns = cudf.concat([patterns, p])
    stats    = cudf.concat([stats,    s])
    
results<jupyter_output>[Reconnaissance (limited)] Mining features...
[Reconnaissance (limited)] Encoding patterns...
[Reconnaissance (limited)] Building models
[Reconnaissance (limited)]	Random Forest
[Reconnaissance (limited)]	Support Vector Machines
[Shellcode (limited)] Mining features...
[Shellcode (limited)] Encoding patterns...
[Shellcode (limited)] Building models
[Shellcode (limited)]	Random Forest
[Shellcode (limited)]	Support Vector Machines
[Analysis (limited)] Mining features...
[Analysis (limited)] Encoding patterns...
[Analysis (limited)] Building models
[Analysis (limited)]	Random Forest
[Analysis (limited)]	Support Vector Machines
[Backdoor (limited)] Mining features...
[Backdoor (limited)] Encoding patterns...
[Backdoor (limited)] Building models
[Backdoor (limited)]	Random Forest
[Backdoor (limited)]	Support Vector Machines
[DoS (limited)] Mining features...
[DoS (limited)] Encoding patterns...
[DoS (limited)] Building models
[DoS (limited)]	Random Forest
[DoS (limited)]	Support Vector Mach[...]<jupyter_text>Results -- simple encoding<jupyter_code>def to_list(x):
    return list(x)

def ranges(bin_no, bins):
    if bin_no == len(bins):
        return '>{0:f}'.format(bins[-1])
    else:
        return '<{0:,f}, {1:,f})'.format(bins[bin_no-1], bins[bin_no])

quantile_bins = pd.DataFrame(quantiles, columns=['feat', 'bins'])
patterns_rec = patterns.merge(feature_encoding, on='feature_enc')

patterns_host = patterns_rec[['pattern_id', 'feature', 'label']].to_pandas()
patterns_host['feat']    = patterns_host.apply(lambda row: row['feature'].split('='), axis = 1)
patterns_host['bin']     = patterns_host.apply(lambda row: row['feat'][1], axis = 1)
patterns_host['feat']    = patterns_host.apply(lambda row: row['feat'][0][:-4], axis = 1)
patterns_host            = patterns_host.merge(quantile_bins, on=['feat'])
patterns_host['ranges']  = patterns_host.apply(lambda row: ranges(int(row['bin']), row['bins']), axis=1)
patterns_host['feature'] = patterns_host['feat'] + '=' + patterns_host['ranges']
patterns_host            = patterns_host[['label', 'pattern_id', 'feature']].sort_values(by='pattern_id')

patterns_host_agg               = patterns_host.groupby(['label', 'feature']).agg({'pattern_id': to_list}).reset_index()
patterns_host_agg['pattern_id'] = patterns_host_agg.apply(lambda row: ','.join([str(e) for e in row['pattern_id']]), axis = 1)#
patterns_host_agg               = patterns_host_agg.groupby(['label', 'pattern_id']).agg({'feature': to_list}).reset_index()

# patterns_host_agg.to_dict('records')

# quantile_bins
stats<jupyter_output><empty_output><jupyter_text>Logistic Regression model<jupyter_code>reg = LogisticRegression(fit_intercept=False, C=0.1)
reg.fit(X, y)

encoded_testing = encoded_testing.drop('pred')
encoded_testing['pred'] = reg.predict(encoded_testing[['col_' + str(i) for i in range(len(patterns_to_encode))]])
results = encoded_testing.groupby(['label', 'pred']).agg({'id': 'count'}).reset_index()
results
calculate_metrics(results)<jupyter_output><empty_output><jupyter_text>Support Vector Machines<jupyter_code>reg = SVC()
reg.fit(X, y)

encoded_testing = encoded_testing.drop('pred')
encoded_testing['pred'] = reg.predict(encoded_testing[['col_' + str(i) for i in range(len(patterns_to_encode))]])
results = encoded_testing.groupby(['label', 'pred']).agg({'id': 'count'}).reset_index()
results
calculate_metrics(results)<jupyter_output><empty_output><jupyter_text>Binned features<jupyter_code># training_df.head()
X = training_df[[col for col in training_df.columns if col not in ['id', 'label', 'attack_id']]]

for col in X:
    X[col] = X[col].astype('float32')

y = training_df['label'].astype('float32')
X_testing = testing_binned[[col for col in testing_binned.columns if col not in ['id', 'test', 'label', 'attack_id']]]
y_testing = testing_binned[['id', 'label']]

reg = SVC()
reg.fit(X, y)
X.columns


for col in X_testing:
    X_testing[col] = X_testing[col].astype('float32')
    
# X_testing.columns
y_testing['pred'] = reg.predict(X_testing)
results = y_testing.groupby(['label', 'pred']).agg({'id': 'count'}).reset_index()
results
calculate_metrics(results)
encoded_df.groupby('col_0').agg({'id': 'count', 'label': 'sum'})
sub = df_binned.query('ct_dst_sport_ltm_bin == 4')[['id', 'label']]
sub['pred'] = 1

sub['label'].sum(),sub['id'].count()
# sub
fff = df_binned.merge(sub, on='id', how='left')#.head()
fff['pred'] = fff['pred'].fillna(0)

fff.groupby('pred').agg({'id': 'count', 'label_x': 'sum'})<jupyter_output><empty_output>