# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + id="wuHzJdk2-NtS" colab_type="code" colab={}
# !pip install -U -q PyDrive
from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
from google.colab import auth
from oauth2client.client import GoogleCredentials

# + id="_QXTQ7Rk-pWP" colab_type="code" colab={}
auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
drive = GoogleDrive(gauth)

# + id="ciaLs228cq9M" colab_type="code" colab={}
downloaded = drive.CreateFile({'id':'13y28pFPbulmcrZHh77kq3fFJXxYfxnch'}) # replace the id with id of file you want to access
downloaded.GetContentFile('train.csv') 
downloaded = drive.CreateFile({'id':'1UmL3P6Ig2Gq0DoyGJRVfQRYCQbSs7SMS'}) # replace the id with id of file you want to access
downloaded.GetContentFile('test.csv')

# + id="RtnKscTnProZ" colab_type="code" colab={}
downloaded = drive.CreateFile({'id':'1xb6RClnre3BF_U-LawftSpMP_Ej37j4f'}) # replace the id with id of file you want to access
downloaded.GetContentFile('sample.csv')

# + id="e0xTqglx_bZw" colab_type="code" colab={}
import pandas as pd
train=pd.read_csv('train.csv') 
test=pd.read_csv('test.csv')

# + id="0gO7KNgi_pTw" colab_type="code" colab={}
train.head()
sample=pd.read_csv('sample.csv')

# + id="YHDhe8iRMZaZ" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 168} outputId="4cb1dd5a-90de-4eb9-a211-cff4b1e02e29"
train.isnull().sum()

# + id="6S5N9dkC_4WL" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 335} outputId="6bb62b94-588c-4cf3-b04f-8dfb51b09550"
import matplotlib.pyplot as plt

plt.subplot(2, 3, 1)
plt.hist(train['toxic'],bins=10)

plt.subplot(2, 3, 2)
plt.hist(train['severe_toxic'],bins=10)

plt.subplot(2, 3, 3)
plt.hist(train['obscene'],bins=10)

plt.subplot(2, 3, 4)
plt.hist(train['threat'],bins=10)

plt.subplot(2, 3, 5)
plt.hist(train['insult'],bins=10)

plt.subplot(2, 3, 6)
plt.hist(train['identity_hate'],bins=10)

# + id="QGfjNso5LVjw" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 296} outputId="2f4017da-2ed8-42e0-b4a1-71037576a34c"
leng=0
length=[(leng+len(x)) for x in train['comment_text']]
plt.hist(length,bins=10)
plt.xlabel('length of words')
plt.ylabel('frequency')

# + id="szpxoY0CUIwo" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 101} outputId="cec4d29d-e50e-4bfb-c8b6-93de69c5eb11"
import pandas as pd
col1="toxic"
col2="severe_toxic"
confusion_matrix = pd.crosstab(train[col1], train[col2])
print("Confusion matrix between toxic and severe toxic:")
print(confusion_matrix)
# new_corr=cramers_corrected_stat(confusion_matrix)
# print("The correlation between Toxic and Severe toxic using Cramer's stat=",new_corr)

# + id="QohaY_Q4MnAj" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 84} outputId="4e8f827d-ee38-4b63-df36-5bdc348d2761"
import re
from nltk.stem import WordNetLemmatizer
from nltk.corpus import stopwords
import nltk
nltk.download('stopwords')
import nltk
nltk.download('wordnet')

stop_words=set(stopwords.words("english"))
lemmatizer=WordNetLemmatizer()

def clean_text(text):
  text = re.sub(r'[^\w\s]','',text, re.UNICODE)
  text = text.lower()
  text = [lemmatizer.lemmatize(token) for token in text.split(" ")]
  text = [lemmatizer.lemmatize(token, "v") for token in text]
  text = [word for word in text if not word in stop_words]
  text = " ".join(text)
  return text

train['Processed_Reviews'] = train.comment_text.apply(lambda x: clean_text(x))

# + id="_6XpvQMPUgN_" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 195} outputId="75e30273-a59e-47ba-dff0-a54aa116b460"
train.head()

# + id="Gx2ULBKBvell" colab_type="code" colab={}
# max_features = 20000 
# maxlen = 100 

# list_sentences_train = train["comment_text"].fillna("_na_").values
# list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
# y = train[list_classes].values
# list_sentences_test = test["comment_text"].fillna("_na_").values

# tokenizer = Tokenizer(num_words=max_features)
# tokenizer.fit_on_texts(list(list_sentences_train))
# list_tokenized_train = tokenizer.texts_to_sequences(list_sentences_train)
# list_tokenized_test = tokenizer.texts_to_sequences(list_sentences_test)
# X_t = pad_sequences(list_tokenized_train, maxlen=maxlen)
# X_te = pad_sequences(list_tokenized_test, maxlen=maxlen)

# + id="c1pdDbi4waPa" colab_type="code" colab={}
# inp = Input(shape=(maxlen,))
# x = Embedding(max_features, embed_size)(inp)
# x = Bidirectional(LSTM(50, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(x)
# x = GlobalMaxPool1D()(x)
# x = Dense(50, activation="relu")(x)
# x = Dropout(0.1)(x)
# x = Dense(6, activation="sigmoid")(x)
# model = Model(inputs=inp, outputs=x)
# model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])

# + id="y1VWHs96wiF_" colab_type="code" colab={}
# model.fit(X_t, y, batch_size=1024, epochs=1, validation_split=0.1);

# + id="V-xkVcIywq3b" colab_type="code" colab={}
# y_test = model.predict([X_te], batch_size=1024, verbose=1)
# sample[list_classes] = y_test

# + id="A_yYCiE6zSf5" colab_type="code" colab={}
# sample[list_classes] = y_test

# + id="PJ0GXdyjUkmh" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="a6790ed5-38ff-43ad-fcb3-9198c6b253b7"
train.Processed_Reviews.apply(lambda x: len(x.split(" "))).mean()

# + id="c8F-m_HBJbcj" colab_type="code" colab={}
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
y = train[list_classes].values

# + id="RSCUhEPFUqRH" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 188} outputId="9a8ae256-c8e4-457d-c945-d30b5cccb933"
from keras.preprocessing.text import Tokenizer 
from keras.preprocessing.sequence import pad_sequences
from keras.models import Model,Sequential
from keras.layers import Dense,Input,LSTM,Embedding,Dropout,Activation,GRU,Flatten
from keras.layers import Bidirectional,GlobalMaxPool1D
from keras.layers import Convolution1D
from keras import initializers, regularizers, constraints, optimizers, layers
from keras.regularizers import l2


max_features=20000
tokenizer=Tokenizer(num_words=max_features)
tokenizer.fit_on_texts(train['Processed_Reviews'])
list_tokenized_train=tokenizer.texts_to_sequences(train['Processed_Reviews'])


max_len=50
x_train=pad_sequences(list_tokenized_train,maxlen=max_len)
y_train=y

from sklearn.model_selection import train_test_split
x_train,x_test,y_train,y_test=train_test_split(x_train,y_train,test_size=0.1,random_state=42)

embed_size = 128
# model = Sequential()
# model.add(Embedding(max_features, embed_size))
# model.add(Bidirectional(LSTM(32, return_sequences = True)))
# model.add(GlobalMaxPool1D())
# model.add(Dense(20, activation="relu"))
# model.add(Dropout(0.05))
# model.add(Dense(6, activation="sigmoid"))
# model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])

# embedding_matrix = np.random.normal(0.020940498, 0.6441043, (nb_words, embed_size))

model=Sequential()
model.add(Embedding(max_features, embed_size))
model.add(Bidirectional(LSTM(50, return_sequences=True, dropout=0.1, recurrent_dropout=0.1)))
model.add(GlobalMaxPool1D())
model.add(Dense(50, activation="relu"))
model.add(Dropout(0.1))
model.add(Dense(6, activation="sigmoid",kernel_regularizer=l2(0.01), bias_regularizer=l2(0.01)))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])

batch_size=1024
epochs=2
model.fit(x_train,y_train,batch_size=batch_size,epochs=epochs,validation_data=(x_test, y_test))

# + id="2jsWJ6CfZE1S" colab_type="code" colab={}


# inp = Input(shape=(maxlen,))
# x = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)
# x = Bidirectional(LSTM(50, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(x)
# x = GlobalMaxPool1D()(x)
# x = Dense(50, activation="relu")(x)
# x = Dropout(0.1)(x)
# x = Dense(6, activation="sigmoid")(x)
# model = Model(inputs=inp, outputs=x)
# model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])

# + id="1DN2jth7WxvG" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 353} outputId="ae02e765-2b04-4a19-8013-bd7531523d6d"
model.summary()

# + id="yf7xarE8M-d5" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 195} outputId="c82f74d1-7947-49bd-ba7e-5a5e52ba86f6"
test.head()

# + id="dEuU2tx0NCgj" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 195} outputId="21d5d944-607c-47dd-aa52-7f749ae62bb2"
test["comment_text"]=test.comment_text.apply(lambda x: clean_text(x))
test.head()

# + id="wi6IZeedKZ56" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="c6234e52-2bd0-411c-ac55-1925e7ded9a1"
max_features=6000
tokenizer=Tokenizer(num_words=max_features)
tokenizer.fit_on_texts(test['comment_text'])
list_tokenized_train=tokenizer.texts_to_sequences(test['comment_text'])


max_len=50
x_test=pad_sequences(list_tokenized_train,maxlen=max_len)
y_test = model.predict([x_test], batch_size=100, verbose=1)

# + id="zuqeR8eoO8_t" colab_type="code" colab={}
sample[list_classes] = y_test

# + id="3O3SSJiaXLUF" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 402} outputId="7d9b516f-2cc0-438f-de1a-84692212a4a0"
sample.head(630)

# + id="tIdVlWaC3azI" colab_type="code" colab={}
ids = sample['id'].str()

submission = pd.DataFrame({'id': ids, 'toxic':sample['toxic'],'severe_toxic':sample['severe_toxic'],'obscene':sample['obscene'],'threat':sample['threat'],'insult':sample['insult'],'identity_hate':sample['identity_hate']})

# + id="Azk9BKpD4Mi_" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 195} outputId="227cb7ee-035b-4ef2-968b-0bce5fc0ce37"
submission.head()

# + id="YpbZ9PsyFFJq" colab_type="code" colab={}
sample.to_csv("sample.csv")

# + id="OBuhGT8i5sx8" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="c7909306-93a1-4d0d-98be-86f166586ff3"
sample.shape

# + id="Fw__NJllL-q5" colab_type="code" colab={}


# + id="iD1Du2voQjYy" colab_type="code" colab={}
from google.colab import files
files.download("submission.csv")

StarcoderdataJupyterScriptsDedupFiltered