# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Define Global Variables and Helper Functions

# +
# Global Imports
import numpy as np
from sklearn import metrics
import pandas as pd
import os
import matplotlib.pyplot as plt

# Helpers
TRAIN_FILE = "resources/train/train.csv"
TEST_FILE = "resources/test/test_with_solutions.csv"
BAD_WORDS_FILE = "resources/badwords.txt"
NO_INSULT = 'NoInsult'
INSULT = 'Insult'

# Comments preprocessing
def preprocess_comment(comment):
    import re
    comment = comment.strip().strip('"')
    comment = comment.replace('_', ' ')
    comment = comment.replace("\\\\", "\\")
    comment = comment.replace('\\n', ' ')
    comment = comment.replace('\\n', ' ')
    comment = comment.lower()
    comment = re.sub(r'^https?:\/\/.*[\r\n]*', 'URL', comment, flags=re.MULTILINE)
    comment = comment.encode('utf-8').decode('unicode-escape')
    return comment

# Predictions Accuracy Report
def predictions_report(pred, ground_truth):
    print("Accuracy: " + str(np.mean(pred == ground_truth)) + "\n")
    print(metrics.classification_report(ground_truth, pred,
        target_names=['NoInsult', 'Insult']))

# Predictions Proba Accuracy Report
def predictions_report_proba(pred, ground_truth, threshold=0.5):
    # Extract Plain Estimations

    labels = []
    for index, prediction in enumerate(pred):
        if isinstance(prediction, list) or type(prediction) is np.ndarray:
            if len(prediction) == 1:
                if prediction[0] >= threshold:
                    labels.append(1)
                else:
                    labels.append(0)
            else:
                if prediction[0] > prediction[1]:
                    labels.append(0)
                else:
                    labels.append(1)
        else:
            if prediction >= threshold:
                labels.append(1)
            else:
                labels.append(0)

    labels = np.array(labels)

    print("Accuracy: " + str(np.mean(labels==ground_truth)) + "\n")
    print(metrics.classification_report(ground_truth, labels,
        target_names=['NoInsult', 'Insult']))

# Load Bad Words Helper
def load_bad_words(badwords_file):
    with open(badwords_file) as f:
        lines = f.readlines()
        return [badword[:-1] for badword in lines]


# -

# # Load Resources

# +
# Load list of badwords
badwords = load_bad_words(BAD_WORDS_FILE)

# Load Train Data
train_df = pd.read_csv(TRAIN_FILE)

# Load Test Data
test_df = pd.read_csv(TEST_FILE)

raw_train_comments = train_df['Comment'].apply(lambda comment: preprocess_comment(comment))
raw_test_comments = test_df['Comment'].apply(lambda comment: preprocess_comment(comment))
# -

print(len(train_df.index))
train_df.head()

# Categories Balance
x = [NO_INSULT, INSULT]
ni_count, icount = train_df['Insult'].value_counts()
y = [ni_count, icount]
plt.barh(x, y)
for index, value in enumerate(y):
    plt.text(value, index, str(value))

train_df['Insult'].value_counts(normalize=True)

print(len(test_df.index))
test_df.head()

ni_count, icount = test_df['Insult'].value_counts()
y = [ni_count, icount]
plt.barh(x, y)
for index, value in enumerate(y):
    plt.text(value, index, str(value))

# Test Dataset Balance
test_df['Insult'].value_counts(normalize=True)

train_labels = train_df["Insult"]
train_comments = train_df["Comment"]

# + pycharm={"name": "#%%\n"}
test_labels = test_df["Insult"]
test_comments = test_df["Comment"]
# -

assert len(train_labels) == len(train_comments) == 3947

# + [markdown] pycharm={"name": "#%% md\n"}
# # NLTK: Corpus Stats
#
# ## Raw Corpus
#
# First, let's take a look to the Frequency Distribution and TF-IDF distribution of the training set without any preprocessing.
#
# Note: the tokenizer applied remove stopwords, normalize the text to lowercase and filter tokens of size < 3 characters

# + pycharm={"name": "#%%\n"}
from feature_extraction import corpus_stats, tf_idf_stats

corpus_stats(train_comments)
# -

tf_idf_stats(train_comments)

# + [markdown] pycharm={"name": "#%% md\n"}
# ## Pre-process comments
# -

train_comments = train_comments.apply(lambda comment: preprocess_comment(comment))
corpus_stats(train_comments)

tf_idf_stats(train_comments)

# + pycharm={"name": "#%%\n"}
test_comments = test_comments.apply(lambda comment: preprocess_comment(comment))
# -

print(len(train_comments), len(test_comments))

# ## Replace Insults
#
# We are going to replace any appearance of an insult with a 'fakeinsult' keyword. The intention of this is to concentrate the frequency of any insult into the same token

# + pycharm={"name": "#%%\n"}
from feature_extraction import replace_badwords

train_comments = train_comments.apply(lambda comment: replace_badwords(comment, badwords))
test_comments = test_comments.apply(lambda comment: replace_badwords(comment, badwords))

corpus_stats(train_comments)
# -

# **Let's check which percentage of comments of each class contains an insult**

# + pycharm={"name": "#%%\n"}
train_df['Comment'] = train_comments
train_df[train_df['Comment'].str.contains('fakeinsult')]['Insult'].value_counts(normalize=True)
# -

#

# # A First Attempt. Naive Classifier using insult presence
#
# As baseline, we are going to use a naive classifier which will classify a comment as offensive if the comment contains an insult keyword

from sklearn.metrics import plot_confusion_matrix
from sklearn.base import is_classifier
from training import NaiveClassifier
naive_classifier = NaiveClassifier(badwords)
naive_predictions = naive_classifier.predict(test_comments)
print("\nNaive Model Result\n")
predictions_report(naive_predictions, test_labels)
plot_confusion_matrix(naive_classifier, test_comments, test_labels, display_labels=[NO_INSULT, INSULT])
plt.show()

# As suspected from the outcome of the previous section, there is a huge number of false positives (35,7%) for NoInsult class as many comments containing insults keywords are not really offenses to a particular user. We will need to work that out in the following approaches

# # A Simple TF-IDF based classifier

from training import train_tfidf
tf_idf_classifier = train_tfidf(train_comments, train_labels)
predictions = tf_idf_classifier.predict(test_comments)
print("\nTF-IDF Model Result\n")
predictions_report(predictions, test_labels)
plot_confusion_matrix(tf_idf_classifier, test_comments, test_labels, display_labels=[NO_INSULT, INSULT])
plt.show()

# Using a simple TF-IDF based classifier (using stochastic gradient descent estimator) improves the global accuracy a lot and, in particular, the model seems to fix the NoInsult category prediction. But now we have more than 50% of false negatives for Insult category.
#
# ## Working with unbalanced training sets
#
# Following https://scikit-learn.org/stable/auto_examples/svm/plot_separating_hyperplane_unbalanced.html we are going to try to tweak the SGD classifier by assigning different weights to each class. Most of the models in scikit-learn have a parameter class_weight. This parameter will affect the computation of the loss in linear model or the criterion in the tree-based model to penalize differently a false classification from the minority and majority class.

tf_idf_classifier = train_tfidf(train_comments, train_labels, class_weight={0: 1, 1: 2})
predictions = tf_idf_classifier.predict(test_comments)
print("\nTF-IDF Model Result\n")
predictions_report(predictions, test_labels)
plot_confusion_matrix(tf_idf_classifier, test_comments, test_labels, display_labels=[NO_INSULT, INSULT])
plt.show()

# # A Custom Classifier
#
# We can continue playing with the class weights and other classifier's parameters, but eventually we will realize that we need to capture more semantics of the problem in the shape of features. We need specific features to capture when a comment is being offensive for another user.
#
# ### A grammar to detect insults to another user

pd.set_option('display.max_colwidth', -1)
train_df[train_df['Insult'] == 1].head(20)

# +
import spacy
from spacy.matcher import Matcher

nlp = spacy.load("en_core_web_sm")
matcher = Matcher(nlp.vocab)

pattern1 = [{"LEMMA": "-PRON-", "LOWER": {"IN": ["you", "your"]}},
            {"LEMMA": {"IN": ["be", "sound"]}},{"OP": "*", "LENGTH": {"<=": 10}},
            {"LOWER": "fakeinsult"}]
matcher.add("insult1", None, pattern1)

pattern2 = [{"LEMMA": "-PRON-", "LOWER": {"IN": ["you", "your"]}},
            {"OP": "*", "LENGTH": {"<=": 4}},
            {"LOWER": "fakeinsult"}]
matcher.add("insult2", None, pattern2)

doc = nlp("Either you are fake or extremely fakeinsult...maybe both...")
matches = matcher(doc)
for match_id, start, end in matches:
    string_id = nlp.vocab.strings[match_id]  # Get string representation
    span = doc[start:end]  # The matched span
    print(match_id, string_id, start, end, span.text)
    
doc = nlp("your such a  fakeinsult head...")
matches = matcher(doc)
for match_id, start, end in matches:
    string_id = nlp.vocab.strings[match_id]  # Get string representation
    span = doc[start:end]  # The matched span
    print(match_id, string_id, start, end, span.text)


# +
insults = train_df[train_df['Insult'] == 1]['Comment']
total_matches = 0
for insult in insults:
    doc = nlp(insult)
    matches = matcher(doc)
    if len(matches) > 0:
        total_matches += 1
        
print("Total offensive comments %d" % len(insults))
print("Total with pattern %d" % total_matches)
# -

# ### Custom Features
#
# - **n_words**: total number of words (tokens) in the comment
# - **n_chars**: total number of characters in the comment
# - **n_dwords**: total number of words in the comment that appears in an English dictionary
# - **you_re**: number of insults patterns matches found in the comment
# - **!**: number of exclamation symbols
# - **allcaps**: number of uppercase characters
# - **@**: number of 'addressing' symbols
# - **bad_ratio**: ratio of insults used in the comment
# - **n_bad**: number of insults in the comment
# - **capsratio**: ratio of uppercased characters
# - **dicratio**: ratio of dictionary words in the comment
# - **sent**: lexicon sentiment score
#

from training import train_custom
custom_classifier = train_custom(train_comments, train_labels)
predictions = custom_classifier.predict(test_comments)
print("\nCustom Model Result\n")
predictions_report(predictions, test_labels)
plot_confusion_matrix(custom_classifier, test_comments, test_labels, display_labels=[NO_INSULT, INSULT])
plt.show()

# # Ensemble Voting Classifier
#
# The goal of ensemble methods is to combine the predictions of several base estimators built with a given learning algorithm in order to improve generalizability / robustness over a single estimator.
#
# We are going to combine the TF-IDF classifier and the custom classifier using a majority vote classifier. Most of the ensemble methods build several instances of a black-box estimator on random subsets of the original training set and then aggregate their individual predictions to form a final prediction. In this case, we need to combine estimators of different nature. Sklearn [Voting classifier](https://scikit-learn.org/stable/modules/ensemble.html#voting-classifier) can help in this situation. The idea behind the VotingClassifier is to combine conceptually different machine learning classifiers and use a majority vote or the average predicted probabilities (soft vote) to predict the class labels. Such a classifier can be useful for a set of equally well performing model in order to balance out their individual weaknesses.

from training import train_assembling_voting
voting_classifier = train_assembling_voting(train_comments, train_labels)
predictions = voting_classifier.predict_proba(test_comments)
print("\nMajority Vote Model Result\n")
predictions_report_proba(predictions, test_labels)
plot_confusion_matrix(voting_classifier, test_comments, test_labels, display_labels=[NO_INSULT, INSULT])
plt.show()

# # Feature Union
#
# To finish with classic estimators based solutions, we are going to make a final attempt by merging the features for both classifiers making use of sklearn [FeatureUnion](https://scikit-learn.org/stable/modules/generated/sklearn.pipeline.FeatureUnion.html). This estimator applies a list of transformer objects in parallel to the input data, then concatenates the results. This is useful to combine several feature extraction mechanisms into a single transformer.

from training import train_feature_union
fu_classifier = train_feature_union(train_comments, train_labels)
predictions = fu_classifier.predict(test_comments)
print("\nFeature Union Model Result\n")
predictions_report(predictions, test_labels)
plot_confusion_matrix(fu_classifier, test_comments, test_labels, display_labels=[NO_INSULT, INSULT])
plt.show()

# # Word Embeddings

# Word Embedding is the collective name for feature learning techniques where words from the vocabulary are mapped to vectors of real numbers. These vectors are calculated from the probability distribution for each word appearing before or after another. To put it another way, words of the same context usually appear together in the corpus, so they will be close in the vector space as well.

import gensim
from feature_extraction import word2vec_model, tokenize_document
raw_tokenized_corpus = [tokenize_document(comment) for comment in raw_train_comments]
dim = 300
max_len = 15
raw_w2v_model = word2vec_model(raw_tokenized_corpus, n_dim=dim)

print(raw_w2v_model.wv.similarity('retarded', 'loser'))
print(raw_w2v_model.wv.most_similar('retarded', topn=20))

tokenized_corpus = [tokenize_document(comment) for comment in train_comments]
test_tokenized_corpus = [tokenize_document(comment) for comment in test_comments]
w2v_model = word2vec_model(tokenized_corpus, n_dim=dim)
w2v_model.wv.most_similar('fakeinsult', topn=20)

# Repeat the process for test corpus
test_raw_tokenized_corpus = [tokenize_document(comment) for comment in raw_test_comments]

# ## Word2Vec based classifier with Keras

## tokenize text for Keras network
import tensorflow as tf
ktokenizer = tf.keras.preprocessing.text.Tokenizer(lower=True, split=' ', 
                     oov_token="NaN", 
                     filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n')
ktokenizer.fit_on_texts(raw_tokenized_corpus)
dic_vocabulary = ktokenizer.word_index
## create sequence
lst_text2seq= ktokenizer.texts_to_sequences(raw_tokenized_corpus)
## padding sequence
X_train = tf.keras.preprocessing.sequence.pad_sequences(lst_text2seq, 
                    maxlen=max_len)

# +
## text to sequence with the fitted tokenizer
lst_text2seq = ktokenizer.texts_to_sequences(test_raw_tokenized_corpus)

## padding sequence
X_test = tf.keras.preprocessing.sequence.pad_sequences(lst_text2seq, maxlen=max_len,
             padding="post", truncating="post")
# -

# Prepare the matrix of embeddings for the embeddings layer
embeddings = np.zeros((len(dic_vocabulary)+1, dim))
for word,idx in dic_vocabulary.items():
    ## update the row with vector
    try:
        embeddings[idx] =  raw_w2v_model.wv[word]
    ## if word not in model then skip and the row stays all 0s
    except:
        pass

# +
# Build the Network
from keras import layers, models
import keras

x_in = layers.Input(shape=(max_len,))

#Embedding
x = layers.Embedding(input_dim=embeddings.shape[0],  
                     output_dim=embeddings.shape[1], 
                     weights=[embeddings],
                     input_length=max_len, trainable=False)(x_in)

## LSTM
x = layers.Bidirectional(layers.LSTM(max_len, dropout=0.2, return_sequences=True))(x)
x = layers.Bidirectional(layers.LSTM(max_len, dropout=0.2))(x)

x = layers.Dense(64, activation='relu')(x)
y_out = layers.Dense(1, activation='sigmoid')(x)

## compile
model = models.Model(x_in, y_out)              
model.compile(loss='binary_crossentropy',
              optimizer=keras.optimizers.Adam(), metrics=['accuracy'])

model.summary()
# -

# Train the model
callbacks = [tf.keras.callbacks.EarlyStopping(monitor='accuracy', patience=3)]
training = model.fit(x=X_train, y=train_labels, batch_size=64, 
                     epochs=20, validation_split=0.3, class_weight={0: 1., 1: 2.5}, callbacks=callbacks)


# Evaluate Model
predictions = model.predict(X_test)
predictions_report_proba(predictions, test_labels)

# ## Same Classifier but replacing insults with 'fakeinsult'

# +
ktokenizer = tf.keras.preprocessing.text.Tokenizer(lower=True, split=' ', 
                     oov_token="NaN", 
                     filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n')
ktokenizer.fit_on_texts(tokenized_corpus)
dic_vocabulary = ktokenizer.word_index
## create sequence
lst_text2seq= ktokenizer.texts_to_sequences(tokenized_corpus)
## padding sequence
X_train = tf.keras.preprocessing.sequence.pad_sequences(lst_text2seq, 
                    maxlen=max_len)

## text to sequence with the fitted tokenizer
lst_text2seq = ktokenizer.texts_to_sequences(test_tokenized_corpus)

## padding sequence
X_test = tf.keras.preprocessing.sequence.pad_sequences(lst_text2seq, maxlen=max_len,
             padding="post", truncating="post")

# Prepare the matrix of embeddings for the embeddings layer
embeddings = np.zeros((len(dic_vocabulary)+1, dim))
for word,idx in dic_vocabulary.items():
    ## update the row with vector
    try:
        embeddings[idx] =  w2v_model.wv[word]
    ## if word not in model then skip and the row stays all 0s
    except:
        pass
    
x_in = layers.Input(shape=(max_len,))

#Embedding
x = layers.Embedding(input_dim=embeddings.shape[0],  
                     output_dim=embeddings.shape[1], 
                     weights=[embeddings],
                     input_length=max_len, trainable=False)(x_in)

## LSTM
x = layers.Bidirectional(layers.LSTM(max_len, dropout=0.2, return_sequences=True))(x)
x = layers.Bidirectional(layers.LSTM(max_len, dropout=0.2))(x)

x = layers.Dense(64, activation='relu')(x)
y_out = layers.Dense(1, activation='sigmoid')(x)

## compile
model = models.Model(x_in, y_out)              
model.compile(loss='binary_crossentropy',
              optimizer=keras.optimizers.Adam(), metrics=['accuracy'])

model.summary()

callbacks = [tf.keras.callbacks.EarlyStopping(monitor='accuracy', patience=3)]
training = model.fit(x=X_train, y=train_labels, batch_size=64, 
                     epochs=20, validation_split=0.3, class_weight={0: 1., 1: 2.5}, callbacks=callbacks)

# Evaluate Model
predictions = model.predict(X_test)
predictions_report_proba(predictions, test_labels)
# -

# # Language Models. BERT
#
# In the field of computer vision, researchers have repeatedly shown the value of transfer learning — pre-training a neural network model on a known task, for instance ImageNet, and then performing fine-tuning — using the trained neural network as the basis of a new purpose-specific model. In recent years, researchers have been showing that a similar technique can be useful in many natural language tasks.
#
# BERT makes use of Transformer, an attention mechanism that learns contextual relations between words (or sub-words) in a text. In its vanilla form, Transformer includes two separate mechanisms — an encoder that reads the text input and a decoder that produces a prediction for the task. Since BERT’s goal is to generate a language model, only the encoder mechanism is necessary.
#
# As opposed to directional models, which read the text input sequentially (left-to-right or right-to-left), the Transformer encoder reads the entire sequence of words at once. Therefore it is considered bidirectional, though it would be more accurate to say that it’s non-directional. This characteristic allows the model to learn the context of a word based on all of its surroundings (left and right of the word).

# +
import transformers
## bert tokenizer
tokenizer = transformers.BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)
maxlen = 50

tokens_train = tokenizer.batch_encode_plus(
    train_comments,
    add_special_tokens = True, # add [CLS], [SEP]
    max_length = maxlen,
    pad_to_max_length=True, # add [PAD] tokens
    truncation=True,
    return_attention_mask = True, # add attention mask to not focus on pad tokens
)

## feature matrix
X_train = [np.asarray(tokens_train['input_ids'], dtype='int32'), 
           np.asarray(tokens_train['attention_mask'], dtype='int32'), 
           np.asarray(tokens_train['token_type_ids'], dtype='int32')]

tokens_test = tokenizer.batch_encode_plus(
    test_comments,
    add_special_tokens = True, # add [CLS], [SEP]
    max_length = maxlen,
    pad_to_max_length=True, # add [PAD] tokens
    truncation=True,
    return_attention_mask = True, # add attention mask to not focus on pad tokens
)

## feature matrix
X_test = [np.asarray(tokens_test['input_ids'], dtype='int32'), 
           np.asarray(tokens_test['attention_mask'], dtype='int32'), 
           np.asarray(tokens_test['token_type_ids'], dtype='int32')]

# +
## inputs
idx = layers.Input((50), dtype="int32", name="input_idx")
masks = layers.Input((50), dtype="int32", name="input_masks")
segments = layers.Input((50), dtype="int32", name="input_segments")

## pre-trained bert
bert_model = transformers.TFBertModel.from_pretrained("bert-base-uncased")
bert_model.trainable = False
bert_out, _ = bert_model([idx, masks, segments])

## fine-tuning
# Add trainable layers on top of frozen layers to adapt the pretrained features on the new data.
bi_lstm = tf.keras.layers.Bidirectional(
        tf.keras.layers.LSTM(64, return_sequences=True)
    )(bert_out)
# Applying hybrid pooling approach to bi_lstm sequence output.
avg_pool = tf.keras.layers.GlobalAveragePooling1D()(bi_lstm)
max_pool = tf.keras.layers.GlobalMaxPooling1D()(bi_lstm)
concat = tf.keras.layers.concatenate([avg_pool, max_pool])
dropout = tf.keras.layers.Dropout(0.3)(concat)
#x = layers.GlobalAveragePooling1D()(bert_out)
#x = layers.Dense(64, activation="relu")(x)
y_out = layers.Dense(1, activation='sigmoid')(dropout)
## compile
model = models.Model([idx, masks, segments], y_out)
for layer in model.layers[:4]:
    layer.trainable = False
model.compile(loss='binary_crossentropy', 
              optimizer='adam', metrics=['accuracy'])
model.summary()
# -

training = model.fit(x=X_train, y=train_labels, batch_size=64, 
                     epochs=1, validation_split=0.3, class_weight={0: 1., 1: 2.5})


# Evaluate Model
predictions = model.predict(X_test)
predictions_report_proba(predictions, test_labels)



StarcoderdataJupyterScriptsDedupFiltered