# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] papermill={"duration": 0.01538, "end_time": "2021-02-01T04:58:21.655913", "exception": false, "start_time": "2021-02-01T04:58:21.640533", "status": "completed"} tags=[] id="OFvFuyyiPrqH"
# ## Import Libraries

# + papermill={"duration": 2.87553, "end_time": "2021-02-01T04:58:24.545749", "exception": false, "start_time": "2021-02-01T04:58:21.670219", "status": "completed"} tags=[] id="YxtM_LwuPrqQ"
import pandas as pd
import numpy as np

import gensim
from gensim.utils import simple_preprocess
from gensim.parsing.porter import PorterStemmer
from gensim.models import Word2Vec

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report,confusion_matrix

import matplotlib.pyplot as plt
import seaborn as sns

# + colab={"base_uri": "https://localhost:8080/"} id="8USJxBg7Q4dj" outputId="4ae5975e-e936-436d-cbcb-8d588b26e705"
from google.colab import drive

drive.mount('/content/gdrive')

# + colab={"base_uri": "https://localhost:8080/"} id="5y3wKiQYRETn" outputId="9b968371-94f5-4e55-e307-fa6b2c08062d"
# !ls gdrive/MyDrive/ML/

# + papermill={"duration": 1.031796, "end_time": "2021-02-01T04:58:25.592794", "exception": false, "start_time": "2021-02-01T04:58:24.560998", "status": "completed"} tags=[] id="0jU-xB5kPrqS"
# Read dataset
train_df = pd.read_csv("gdrive/MyDrive/ML/Train.csv")

# + papermill={"duration": 0.036974, "end_time": "2021-02-01T04:58:25.644824", "exception": false, "start_time": "2021-02-01T04:58:25.607850", "status": "completed"} tags=[] colab={"base_uri": "https://localhost:8080/", "height": 195} id="q7YBAfySPrqS" outputId="fdb6e36e-58dd-4712-f4bf-223af175a505"
# first few lines
train_df.head()

# + papermill={"duration": 0.023942, "end_time": "2021-02-01T04:58:25.684126", "exception": false, "start_time": "2021-02-01T04:58:25.660184", "status": "completed"} tags=[] colab={"base_uri": "https://localhost:8080/"} id="YrYS7R6yPrqT" outputId="2bdca47e-0678-428c-86e1-13d4abe6bea1"
# shape of dataset
train_df.shape

# + [markdown] papermill={"duration": 0.015097, "end_time": "2021-02-01T04:58:25.714778", "exception": false, "start_time": "2021-02-01T04:58:25.699681", "status": "completed"} tags=[] id="SKlV_y2LPrqU"
# Let's see weather our dataset is balanced or imbalanced

# + papermill={"duration": 0.158348, "end_time": "2021-02-01T04:58:25.890994", "exception": false, "start_time": "2021-02-01T04:58:25.732646", "status": "completed"} tags=[] colab={"base_uri": "https://localhost:8080/", "height": 350} id="vQw-165APrqU" outputId="3f19a37f-8519-4706-8cb5-18e12ca3ff76"
sns.countplot(train_df['label'])

# + [markdown] papermill={"duration": 0.016548, "end_time": "2021-02-01T04:58:25.924439", "exception": false, "start_time": "2021-02-01T04:58:25.907891", "status": "completed"} tags=[] id="4ZuDWFbgPrqV"
# From above graph we can see that our dataset is balanced dataset.So that's great

# + [markdown] papermill={"duration": 0.016561, "end_time": "2021-02-01T04:58:25.957738", "exception": false, "start_time": "2021-02-01T04:58:25.941177", "status": "completed"} tags=[] id="ZKT5srlQPrqV"
# ## Tokenization

# + [markdown] papermill={"duration": 0.016597, "end_time": "2021-02-01T04:58:25.990816", "exception": false, "start_time": "2021-02-01T04:58:25.974219", "status": "completed"} tags=[] id="l20cUX_zPrqV"
# **Tokenization** is the process of tokenizing or splitting a string, text into a list of tokens.  

# + papermill={"duration": 35.389385, "end_time": "2021-02-01T04:59:01.396866", "exception": false, "start_time": "2021-02-01T04:58:26.007481", "status": "completed"} tags=[] id="Ym0wvgv1PrqV"
train_df['tokenized_text'] = [simple_preprocess(line, deacc=True) for line in train_df['text']]

# + [markdown] papermill={"duration": 0.016473, "end_time": "2021-02-01T04:59:01.430797", "exception": false, "start_time": "2021-02-01T04:59:01.414324", "status": "completed"} tags=[] id="iQywj9ajPrqW"
# ## Stemming

# + [markdown] papermill={"duration": 0.016586, "end_time": "2021-02-01T04:59:01.464066", "exception": false, "start_time": "2021-02-01T04:59:01.447480", "status": "completed"} tags=[] id="t8ZTECI9PrqW"
# **Stemming** is the process of reducing a word to its word stem that affixes to suffixes and prefixes or to the roots of words known as a lemma.The input to the stemmer is tokenized words.
# A stemming algorithm reduces the words “chocolates”, “chocolatey”, “choco” to the root word, “chocolate”   

# + papermill={"duration": 59.843074, "end_time": "2021-02-01T05:00:01.323932", "exception": false, "start_time": "2021-02-01T04:59:01.480858", "status": "completed"} tags=[] colab={"base_uri": "https://localhost:8080/", "height": 402} id="lGSH_yThPrqX" outputId="c1036984-ace7-4ebe-a279-a38bd3eb85b6"
porter_stemmer = PorterStemmer()
train_df['stemmed_tokens'] = [[porter_stemmer.stem(word) for word in tokens] for tokens in train_df['tokenized_text']]
train_df


# + [markdown] papermill={"duration": 0.017135, "end_time": "2021-02-01T05:00:01.358373", "exception": false, "start_time": "2021-02-01T05:00:01.341238", "status": "completed"} tags=[] id="eqy9dfqCPrqX"
# ## Train-Test Split

# + papermill={"duration": 0.050324, "end_time": "2021-02-01T05:00:01.425955", "exception": false, "start_time": "2021-02-01T05:00:01.375631", "status": "completed"} tags=[] colab={"base_uri": "https://localhost:8080/"} id="KxjLX3DdPrqX" outputId="665c8be1-8eea-418e-88d1-ac2f8d5e11bc"

# Train Test Split Function
def split_train_test(train_df, test_size=0.3, shuffle_state=True):
    X_train, X_test, Y_train, Y_test = train_test_split(train_df[['stemmed_tokens']], 
                                                        train_df['label'], 
                                                        shuffle=shuffle_state,
                                                        test_size=test_size, 
                                                        random_state=15)
    print("Value counts for Train sentiments")
    print(Y_train.value_counts())
    print("Value counts for Test sentiments")
    print(Y_test.value_counts())
    print(type(X_train))
    print(type(Y_train))
    X_train = X_train.reset_index()
    X_test = X_test.reset_index()
    Y_train = Y_train.to_frame()
    Y_train = Y_train.reset_index()
    Y_test = Y_test.to_frame()
    Y_test = Y_test.reset_index()
    print(X_train.head())
    return X_train, X_test, Y_train, Y_test

# Call the train_test_split
X_train, X_test, Y_train, Y_test = split_train_test(train_df)

# + [markdown] papermill={"duration": 0.017508, "end_time": "2021-02-01T05:00:01.461749", "exception": false, "start_time": "2021-02-01T05:00:01.444241", "status": "completed"} tags=[] id="Y61nHW_IPrqY"
# ## Word2Vec Model Creation

# + [markdown] papermill={"duration": 0.017476, "end_time": "2021-02-01T05:00:01.497168", "exception": false, "start_time": "2021-02-01T05:00:01.479692", "status": "completed"} tags=[] id="A0ogL9EwPrqZ"
# **word2vec** algorithm uses a neural network model to learn word associations from a large corpus of text. Once trained, such a model can detect synonymous words or suggest additional words for a partial sentence. As the name implies, word2vec represents each distinct word with a particular list of numbers called a vector. 

# + papermill={"duration": 117.576898, "end_time": "2021-02-01T05:01:59.091812", "exception": false, "start_time": "2021-02-01T05:00:01.514914", "status": "completed"} tags=[] colab={"base_uri": "https://localhost:8080/", "height": 454} id="0MYGP7IYPrqZ" outputId="56341d3d-4f68-4574-febd-bc44ae4e395f"
size = 500
window = 3
min_count = 1
workers = 3
# 0 for CBOW, 1 for skip-gram
sg = 0
OUTPUT_FOLDER = '/kaggle/working/'
# Function to train word2vec model
def make_word2vec_model(train_df, padding, sg, min_count, size, workers, window):
    if  padding:
        #print(len(train))
        temp_df = pd.Series(train_df['stemmed_tokens']).values
        temp_df = list(temp_df)
        temp_df.append(['pad'])
        #print(str(size))
        word2vec_file = OUTPUT_FOLDER + '2ata' + '_PAD.model'
    w2v_model = Word2Vec(temp_df, min_count = min_count, size = size, workers = workers, window = window, sg = sg)

    #w2v_model.save(word2vec_file)
    return w2v_model, word2vec_file

# Train Word2vec model
w2vmodel, word2vec_file = make_word2vec_model(train_df, padding=True, sg=sg, min_count=min_count, size=size, workers=workers, window=window)

# + [markdown] papermill={"duration": 0.017826, "end_time": "2021-02-01T05:01:59.128451", "exception": false, "start_time": "2021-02-01T05:01:59.110625", "status": "completed"} tags=[] id="2uKaNf21Prqa"
# ## Padding

# + [markdown] papermill={"duration": 0.017726, "end_time": "2021-02-01T05:01:59.164016", "exception": false, "start_time": "2021-02-01T05:01:59.146290", "status": "completed"} tags=[] id="eailXT7kPrqa"
# All the neural networks require to have inputs that have the same shape and size. However, when we pre-process and use the texts as inputs for our model e.g. LSTM, not all the sentences have the same length. In other words, naturally, some of the sentences are longer or shorter. We need to have the inputs with the same size, this is where the padding is necessary.
# **Padding** will make all sentences of same length by inserting 0 in the end or bigenning of the sentences

# + papermill={"duration": 0.044877, "end_time": "2021-02-01T05:01:59.226871", "exception": false, "start_time": "2021-02-01T05:01:59.181994", "status": "completed"} tags=[] id="SZAtDGPbPrqa" outputId="d79fdcc1-fc73-4983-e5db-70a1aad68cae"
max_sen_len = train_df.stemmed_tokens.map(len).max()

padding_idx = w2vmodel.wv.vocab['pad'].index
print(padding_idx)
def make_word2vec_vector_cnn(sentence):
    padded_X = [padding_idx for i in range(max_sen_len)]
    i = 0
    for word in sentence:
        if word not in w2vmodel.wv.vocab:
            padded_X[i] = 0
        else:
            padded_X[i] = w2vmodel.wv.vocab[word].index
        i += 1
    return torch.tensor(padded_X, dtype=torch.long, device=device).view(1, -1)


# + papermill={"duration": 0.025094, "end_time": "2021-02-01T05:01:59.270003", "exception": false, "start_time": "2021-02-01T05:01:59.244909", "status": "completed"} tags=[] id="92QCoM01Prqa" outputId="3b5090fe-db77-45f4-9d12-85389455ce30"
max_sen_len

# + papermill={"duration": 0.025654, "end_time": "2021-02-01T05:01:59.314047", "exception": false, "start_time": "2021-02-01T05:01:59.288393", "status": "completed"} tags=[] id="XCqLqCHSPrqb" outputId="7b92d483-a06a-4995-c8ff-c78de7b45a59"
padding_idx

# + [markdown] papermill={"duration": 0.018659, "end_time": "2021-02-01T05:01:59.351560", "exception": false, "start_time": "2021-02-01T05:01:59.332901", "status": "completed"} tags=[] id="v6ODWVYFPrqb"
# ## CNN model 

# + papermill={"duration": 0.377412, "end_time": "2021-02-01T05:01:59.747977", "exception": false, "start_time": "2021-02-01T05:01:59.370565", "status": "completed"} tags=[] id="jhIsD_u6Prqb" outputId="69b74536-cfe6-4dcc-b416-d46bf7ac0e80"
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
device

# + papermill={"duration": 0.031464, "end_time": "2021-02-01T05:01:59.799039", "exception": false, "start_time": "2021-02-01T05:01:59.767575", "status": "completed"} tags=[] id="_KVKyJ9-Prqb"
EMBEDDING_SIZE = 500
NUM_FILTERS = 10

#torch.nn.Conv2d(in_channels: int, out_channels: int, kernel_size: Union[T, Tuple[T, T]], 
#stride: Union[T, Tuple[T, T]] = 1, padding: Union[T, Tuple[T, T]] = 0, 
#dilation: Union[T, Tuple[T, T]] = 1, groups: int = 1, bias: bool = True, padding_mode: str = 'zeros')

class CnnTextClassifier(nn.Module):
    def __init__(self, vocab_size, num_classes, window_sizes=(1,2,3,5)):
        super(CnnTextClassifier, self).__init__()
        w2vmodel = gensim.models.KeyedVectors.load(OUTPUT_FOLDER + '2ata_PAD.model')
        weights = w2vmodel.wv
        # With pretrained embeddings
        self.embedding = nn.Embedding.from_pretrained(torch.FloatTensor(weights.vectors), padding_idx=w2vmodel.wv.vocab['pad'].index)
        
        # like a python list, it was designed to store any desired number of nn.Module
        self.convs = nn.ModuleList([
                                   nn.Conv2d(1, NUM_FILTERS, [window_size, EMBEDDING_SIZE], padding=(window_size - 1, 0))
                                   for window_size in window_sizes
        ])
    
        self.fc = nn.Linear(NUM_FILTERS * len(window_sizes), num_classes)

    def forward(self, x):
        x = self.embedding(x) # [B, T, E]

        # Apply a convolution + max_pool layer for each window size
        x = torch.unsqueeze(x, 1)
        xs = []
        for conv in self.convs:
            x2 = torch.tanh(conv(x))
            x2 = torch.squeeze(x2, -1)
            x2 = F.max_pool1d(x2, x2.size(2))
            xs.append(x2)
        x = torch.cat(xs, 2)

        # FC
        x = x.view(x.size(0), -1)
        logits = self.fc(x)

        probs = F.softmax(logits, dim = 1)

        return probs


# + papermill={"duration": 0.026038, "end_time": "2021-02-01T05:01:59.844386", "exception": false, "start_time": "2021-02-01T05:01:59.818348", "status": "completed"} tags=[] id="f2Y5Zi3UPrqc"
def make_target(label):
    if label == 0:
        return torch.tensor([0], dtype=torch.long, device=device)
    elif label == 1:
        return torch.tensor([1], dtype=torch.long, device=device)


# + [markdown] papermill={"duration": 0.01963, "end_time": "2021-02-01T05:01:59.883419", "exception": false, "start_time": "2021-02-01T05:01:59.863789", "status": "completed"} tags=[] id="ZbfTWpYBPrqc"
# ## Train the model

# + papermill={"duration": 5.154448, "end_time": "2021-02-01T05:02:05.058040", "exception": false, "start_time": "2021-02-01T05:01:59.903592", "status": "completed"} tags=[] id="uOLwCpUrPrqd" outputId="be36ad59-bac1-4989-a2c0-79e724f7a931"
NUM_CLASSES = 2
VOCAB_SIZE = len(w2vmodel.wv.vocab)
print(VOCAB_SIZE)
cnn_model = CnnTextClassifier(vocab_size=VOCAB_SIZE, num_classes=NUM_CLASSES)
cnn_model.to(device)
loss_function = nn.CrossEntropyLoss()
optimizer = optim.Adam(cnn_model.parameters(), lr=0.0001)
num_epochs = 5

# + papermill={"duration": 1206.785601, "end_time": "2021-02-01T05:22:11.863925", "exception": false, "start_time": "2021-02-01T05:02:05.078324", "status": "completed"} tags=[] id="UBaaSYrxPrqd" outputId="4d362b85-916e-4701-9f76-9dc14d3ecea5"
# Open the file for writing loss
loss_file_name = OUTPUT_FOLDER + '1cnn_class_big_loss_with_padding.csv'
f = open(loss_file_name,'w')
f.write('iter, loss')
f.write('\n')
losses = []

cnn_model.train()
for epoch in range(num_epochs):
    print("Epoch" + str(epoch + 1))
    train_loss = 0

    for index, row in X_train.iterrows():
        # Clearing the accumulated gradients
        cnn_model.zero_grad()

        # Make the bag of words vector for stemmed tokens 
        bow_vec = make_word2vec_vector_cnn(row['stemmed_tokens'])
       
        # Forward pass to get output
        probs = cnn_model(bow_vec)

        # Get the target label
        #print(Y_train['label'][index])
        target = make_target(Y_train['label'][index])

        # Calculate Loss: softmax --> cross entropy loss
        loss = loss_function(probs, target)
        train_loss += loss.item()
        

        # Getting gradients w.r.t. parameters
        loss.backward()

        # Updating parameters
        optimizer.step()

    print(f'train_loss : {train_loss / len(X_train)}')
    print("Epoch ran :"+ str(epoch+1))
    
    f.write(str((epoch+1)) + "," + str(train_loss / len(X_train)))
    f.write('\n')
    train_loss = 0

torch.save(cnn_model, OUTPUT_FOLDER + 'cnn_big_model_500_with_padding.pth')

f.close()
print("Input vector")
print("Probs")
print(probs)
print(torch.argmax(probs, dim=1).cpu().numpy()[0])

# + [markdown] papermill={"duration": 0.021574, "end_time": "2021-02-01T05:22:11.908648", "exception": false, "start_time": "2021-02-01T05:22:11.887074", "status": "completed"} tags=[] id="OK-xJXPxPrqf"
# ## Model Test

# + papermill={"duration": 49.988305, "end_time": "2021-02-01T05:23:01.919057", "exception": false, "start_time": "2021-02-01T05:22:11.930752", "status": "completed"} tags=[] id="zOkIjLXfPrqi" outputId="fcb08291-4a67-4713-cc42-81ab8fcf8805"
bow_cnn_predictions = []
original_lables_cnn_bow = []
cnn_model.eval()
loss_df = pd.read_csv(OUTPUT_FOLDER + '1cnn_class_big_loss_with_padding.csv')
print(loss_df.columns)
# loss_df.plot('loss')

y_pred_list = []
y_true_list = []

with torch.no_grad():
    for index, row in X_test.iterrows():
        #print(row['stemmed_tokens'])
        bow_vec = make_word2vec_vector_cnn(row['stemmed_tokens'])
        #print(bow_vec)
        probs = cnn_model(bow_vec)
        #print(probs.data)
        _, predicted = torch.max(probs.data,  1)
        
        bow_cnn_predictions.append(predicted.cpu().numpy()[0])
        original_lables_cnn_bow.append(make_target(Y_test['label'][index]).cpu().numpy()[0])

print(confusion_matrix(original_lables_cnn_bow, bow_cnn_predictions))
#print(original_lables_cnn_bow)
print(classification_report(original_lables_cnn_bow,bow_cnn_predictions))
loss_file_name = OUTPUT_FOLDER + '1cnn_class_big_loss_with_padding.csv'
loss_df = pd.read_csv(loss_file_name)
print(loss_df.columns)
plt_500_padding_30_epochs = loss_df[' loss'].plot()
fig = plt_500_padding_30_epochs.get_figure()
fig.savefig(OUTPUT_FOLDER + '1loss_plt_500_padding_30_epochs.pdf')

StarcoderdataJupyterScriptsDedupFiltered