# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import numpy as np
import pandas as pd
import nltk
import string
import re
import matplotlib.pyplot as plt
from scipy import stats
from keras.preprocessing import sequence
from keras.layers import  Input, Dense, Flatten, Add,\
    BatchNormalization, Concatenate, Dropout, Activation, Input, Embedding, Conv1D, MaxPooling1D, GlobalMaxPooling1D
from keras.optimizers import Adam
from keras.models import Model, load_model
import tensorflow as tf
from keras import regularizers
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau, TensorBoard, Callback
import keras.backend as K

from sklearn.metrics import roc_auc_score
class ROCCallBack(Callback):
    def __init__(self,training_data,validation_data):
        super().__init__()
        self.x = training_data[0]
        self.y = training_data[1]
        self.x_val = validation_data[0]
        self.y_val = validation_data[1]

    def on_epoch_end(self, epoch, logs={}):
        y_pred = self.model.predict(self.x)
        y_pred_val = self.model.predict(self.x_val)
        roc_train, roc_val = [], []
        for counter, i in enumerate(output_names):            
            roc_train.append(roc_auc_score(self.y[counter], y_pred[counter].flatten()))            
            roc_val.append(roc_auc_score(self.y_val[counter], y_pred_val[counter].flatten())) 
        print()
        print('roc-auc: %s - roc-auc_val: %s' % (str(round(np.mean(roc_train),4)),str(round(np.mean(roc_val),4))))
        print('Val ROCs: ' + '    '.join([output_names[counter] + ': ' + str(round(i,4))for counter, i in enumerate(roc_val)]))


def flatten(x):
    if isinstance(x, (np.ndarray, list, tuple, pd.Series)):
        lst = []
        for i in x:
            lst += flatten(i)
        return lst
    else:
        return [x]


# #### Load Data

train = pd.read_csv('data/train.csv')
test = pd.read_csv('data/test.csv')

# use linear preds as pseudo-labeled data
linear_preds = pd.read_csv('data/answers/linear_ensemble_0.85_09724.csv')

train.head()

train[output_names].apply(pd.value_counts)

output_names = ['toxic','severe_toxic','obscene','threat','insult','identity_hate']

train['tokens'] = train['comment_text'].apply(lambda x: list(filter(lambda z: len(z), map(lambda y: y.lower(), nltk.word_tokenize(x)))))
test['tokens'] = test['comment_text'].apply(lambda x: list(filter(lambda z: len(z), map(lambda y: y.lower(), nltk.word_tokenize(x)))))

train['tokens'].head()

train['tokLength'] = train['tokens'].apply(lambda x: len(x))

train['tokLength'].describe()

train['tokLength'].hist(bins = range(0, 1000, 20),linewidth = 1, edgecolor = 'black' )
plt.show()
#looks like max_length of 500 should be good

stats.percentileofscore(train['tokLength'].values, 500)

max_length = 500

# What are the most common words in the texts?

dist = nltk.FreqDist(flatten(train.tokens.values))

print('Number of total tokens in the training set:', dist.N())
print('Number of unique tokens in the training set:', dist.B())

dist.most_common(n=50)

# #### Load GloVe

f = open('data/glove.42B.300d.txt', 'r', encoding = 'utf-8')

all_unique_tokens = set(flatten(train.tokens.values)).union(set(flatten(test.tokens.values)))

embeddings = {}
for line in f:
    values = line.split()
    word = values[0]
    # Whole GloVe embeddings doesn't fit in my GPU memory, so only take words which appear in data for now. 
    # Can always swap weights for embedding layer after model training
    if word in all_unique_tokens:
        coefs = np.array(values[1:], dtype = 'float32')
        embeddings[word] = coefs

np.random.seed(10)
embeddings['<UNK>'] = np.random.normal(scale = 0.6,size = embeddings['.'].shape)

# What are the most common words not in the vocab?

notInVocab = []
for i in dist.most_common():
    if i[0] not in embeddings.keys() and i[1]>100:
        notInVocab.append((i[0], i[1]))

notInVocab

# #### Data Preparation

idx2word = {count:i for count, i in enumerate(embeddings.keys())}
word2idx = {idx2word[i]: i for i in idx2word.keys()}

print(word2idx['testing'])
print(word2idx['<UNK>'])
print(idx2word[123])

n_factors = 300
vocab_size = len(embeddings)

zero_vector_idx = vocab_size #place blank character last
idx2word[zero_vector_idx] = ''
word2idx[''] = zero_vector_idx


def create_emb():
    emb = np.zeros((vocab_size+1,n_factors), dtype = 'float32')
    for i in range(0, vocab_size):
        word = idx2word[i]
        emb[i,:] = embeddings[word] #each row is a word
    return emb


emb = create_emb()

emb.shape

# +
# parse train/test sets
table = str.maketrans("","", string.punctuation)
def toksToInds(listToks):
    ans = []
    for count, i in enumerate(listToks):
        try:
            ans.append(word2idx[i])
        except KeyError:
            temp = i.translate(table)
            if temp and temp in word2idx.keys():
                ans.append(word2idx[temp])
            else:
                ans.append(word2idx['<UNK>'])        
    return np.array(ans)

train['idxInput'] = train['tokens'].apply(toksToInds)
test['idxInput'] = test['tokens'].apply(toksToInds)
# -

# train test split
np.random.seed(10)
indexTrain = np.random.choice(range(train.shape[0]), size = int(0.9*train.shape[0]), replace = False)
indexVal = list(set(range(train.shape[0])) - set(indexTrain))
traindf = train.loc[indexTrain]
valdf = train.loc[indexVal]

# only add pseudo labeled data to the training set
pseudo = test.merge(linear_preds, how = 'left', on='id').sample(int(train.shape[0]*0.3), random_state = 10).round()
pseudo[output_names] = pseudo[output_names].astype(int)
traindf = traindf.append(pseudo, ignore_index = True).sample(frac=1, random_state = 10).reset_index(drop = True)

dataInputTrain = sequence.pad_sequences(traindf['idxInput'].values, maxlen = max_length, value = zero_vector_idx)
dataInputVal = sequence.pad_sequences(valdf['idxInput'].values, maxlen = max_length, value = zero_vector_idx)
dataInputTest = sequence.pad_sequences(test['idxInput'].values, maxlen = max_length, value = zero_vector_idx)

dataInputTrain.shape

' '.join([idx2word[i] for i in dataInputTrain[0,:]])

valdf[output_names].apply(lambda x: pd.value_counts(np.round(x)))

# #### Modelling

# +
# simple starter
# vec_input = Input(shape = (max_length,))
# x = Embedding(vocab_size+1, n_factors, input_length=max_length,weights = [emb], trainable = False)(vec_input)
# x = Conv1D(64,7,activation = 'relu', padding = 'same')(x)
# x = MaxPooling1D(2)(x)
# # x = Dropout(0.2)(x)

# x = Conv1D(64,7,activation = 'relu', padding = 'same')(x)
# x = GlobalMaxPooling1D()(x)
# x = Dropout(0.5)(x)

# x = Dense(32, activation = 'relu', kernel_regularizer=regularizers.l2(1e-4))(x)
# x = Dense(len(output_names), activation = 'sigmoid')(x)

# model = Model(inputs = vec_input, outputs = x)
# model.compile(loss='binary_crossentropy', optimizer=Adam(0.001), metrics=['accuracy'])
# model.summary()

# +
vec_input = Input(shape = (max_length,))
x = Embedding(vocab_size+1, n_factors, input_length=max_length,weights = [emb], trainable = False)(vec_input)
x = BatchNormalization(axis = -1)(x)
#conv 1
x = Conv1D(32, 7, activation = 'relu', padding = 'same')(x)
x = BatchNormalization(axis = -1)(x)
x = MaxPooling1D(pool_size = 2)(x)
x = Dropout(0.2)(x)

#conv 2
x = Conv1D(64, 7, activation = 'relu', padding = 'same')(x)
x = BatchNormalization(axis = -1)(x)
x = MaxPooling1D(pool_size = 2)(x)
x = Dropout(0.2)(x)

#conv 3
x = Conv1D(64, 7, activation = 'relu', padding = 'same')(x)
x = BatchNormalization(axis = -1)(x)
x = MaxPooling1D(pool_size = 2)(x)
x = Dropout(0.2)(x)

#conv 4
x = Conv1D(128, 7, activation = 'relu', padding = 'same')(x)
x = BatchNormalization(axis = -1)(x)
x = MaxPooling1D(pool_size = 2)(x)
x = Dropout(0.2)(x)

# #conv 5
# x = Conv1D(128, 7, activation = 'relu', padding = 'same')(x)
# x = BatchNormalization(axis = -1)(x)
# x = MaxPooling1D(pool_size = 2)(x)
# x = Dropout(0.2)(x)

#dense 1
x = GlobalMaxPooling1D()(x)
x = Dense(64, activation = 'relu')(x)
x = BatchNormalization(axis = -1)(x)
x = Dropout(0.4)(x)

#dense 2
x = Dense(128, activation = 'relu')(x)
x = BatchNormalization(axis = -1)(x)
x = Dropout(0.4)(x)
# -

# add output layers
outputs = []
for i in output_names:
    intermediate_act = Dense(units = 128, activation = 'relu')(x)
    intermediate_act = BatchNormalization(axis = -1)(intermediate_act)
    if i in ['toxic','obscene']:
        intermediate_act = Dense(units = 512, activation = 'relu')(intermediate_act)
        intermediate_act = BatchNormalization(axis = -1)(intermediate_act)
    outputs.append(Dense(units = 1, activation = 'sigmoid', name = i)(intermediate_act))

model = Model(inputs = vec_input, outputs = outputs)

model.summary()

model.compile(Adam(1e-5), loss = 'binary_crossentropy')

earlyStopping = EarlyStopping(monitor='val_loss', patience=10, verbose=0, mode='min')
mcp_save = ModelCheckpoint('weights/cnn_mdl', save_best_only=True, monitor='val_loss', mode='min')
reduce_lr_loss = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=5, verbose=1, epsilon=1e-4, mode='min')
tensor_board = TensorBoard(log_dir='./logs/run1', write_graph = False,)
roc_callback = ROCCallBack(training_data = [dataInputTrain, [traindf[i] for i in output_names]],
                          validation_data = [dataInputVal, [valdf[i] for i in output_names]])

K.set_value(model.optimizer.lr, 1e-3)
model.fit(x = dataInputTrain, 
          y = [traindf[i] for i in output_names],
         batch_size = 64, epochs = 200, validation_data = [dataInputVal, [valdf[i] for i in output_names]],
         callbacks=[earlyStopping, mcp_save, reduce_lr_loss, tensor_board, roc_callback])

# Examining what the model got wrong

pred_val = model.predict(dataInputVal)

# +
# for count,i in enumerate(output_names):
#     print('---------------' + i + '---------------------')
#     dif = (valdf[i] - pred_val[count].flatten()).abs().sort_values(ascending = False)
#     most_dif = valdf.loc[dif.index[:2]]
#     for count2, j in enumerate(most_dif.iterrows()):
#         print('Predicted', pred_val[count].flatten()[valdf[i].index.get_loc(j[0])]
#               , 'Actual', j[1][i],
#              '\n',
#              j[1]['comment_text'])
# -

# #### Submission

pred = model.predict(test_mat)

for count,i in enumerate(output_names):
    test[i] = pred[count].flatten()

test[['id', 'toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']].to_csv('data/answers/cnn1.csv', index = False)

model_stopped = load_model('weights/cnn_mdl')

pred_stopped = model_stopped.predict(test_mat)

for count,i in enumerate(output_names):
    test[i] = pred[count].flatten()
test[['id', 'toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']].to_csv('data/answers/cnnStopped.csv', index = False)    



StarcoderdataJupyterScriptsDedupFiltered