# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3.9.5 64-bit
#     name: python3
# ---

# + id="30_EgaSf4UDa"
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from sklearn.model_selection import train_test_split
from tensorflow.keras import utils as np_utils
from keras.models import Sequential
from keras.layers import Embedding, LSTM, Dense, Bidirectional
from sklearn.metrics import confusion_matrix, classification_report
from tqdm import tqdm
import re

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1' 
# tf.get_logger().setLevel('INFO')

# + id="0FJETdsk4adu"
train_df = pd.read_json('../../data/raw/raw_nyt/raw_train.json', lines = True)

# + colab={"base_uri": "https://localhost:8080/"} id="uMtiqfBT4jza" outputId="1450c293-d7ed-4b72-c7a8-16f2fadc3796"
print(train_df.columns)

# + colab={"base_uri": "https://localhost:8080/", "height": 293} id="eNodE-ay4tOQ" outputId="8b02f439-6847-4166-9262-c91f8b5c5d56"
train_df.head()


# + id="H3dXrXAq47zE"
#rel_sets -> relation_sets
def extract_relations(df):
  relations = []
  for rel_sets in df['relationMentions']:
    relations.append(rel_sets[-1]['label'])
  return relations

train_df['relations'] = extract_relations(train_df)

# + colab={"base_uri": "https://localhost:8080/"} id="P01rS2Oz5vw9" outputId="1494c750-8287-48ac-d4a4-54fc997cbbaf"
set(train_df['relations'])

# + colab={"base_uri": "https://localhost:8080/"} id="JrL7cLOC52_-" outputId="c618e334-bc00-453d-da1c-f0218bdb83b9"
plt.style.use('dark_background')
train_df['relations'].value_counts().plot.barh(title='Relation Counts', logx = True, figsize=(15,10))
plt.xlabel('Frequencies', fontsize=18)
plt.ylabel('Relations', fontsize=18)
plt.savefig('../../reports/figures/raw_nyt_train_data_relationFrequencies.png')

# + colab={"base_uri": "https://localhost:8080/"} id="94dxDVLE6Ann" outputId="332a1d51-33aa-4137-822d-e6405384f380"
train_df['sentText'].head()


# + id="HZo5BYO96Sxc"
def get_x(df):
  X = []
  for data in df['sentText']:
    X.append(data.lower())
  return X

X = get_x(train_df)

# + colab={"base_uri": "https://localhost:8080/"} id="7Nuae3Hm6vwY" outputId="1cdb62a4-eb5f-4dab-d556-0d9c0a42a763"
for i in range(5):
    print(X[i], end="\n\n")


# + id="n4T1_hGQ7BYL"
def find_max_len(x, max_len):
  if(max_len < len(x.split())):
    max_len = len(x.split())
  return max_len


# + colab={"base_uri": "https://localhost:8080/"} id="pE26byC26xE5" outputId="42537beb-e660-4d8a-d154-82a81a583d9a"
max_len = 0
for x in train_df['sentText']:
  max_len = find_max_len(x, max_len)
print("Max Length = ", max_len)

# + colab={"base_uri": "https://localhost:8080/"} id="yCL2zoj469f4" outputId="40bc5cb7-781c-4bef-9e53-4015a0ecac8c"
n_most_common_words = 65000 # vocabulary size
max_len = 100 # Max Length of Sequence
tokenizer = Tokenizer(num_words=n_most_common_words, filters='!"#$%&()*+,-./:;=?@[]^_`{|}~', lower=True)
tokenizer.fit_on_texts(train_df['sentText'].values)
sequences = tokenizer.texts_to_sequences(train_df['sentText'].values)
word_index = tokenizer.word_index
print('Found %s unique tokens.' % len(word_index))
X = pad_sequences(sequences, maxlen=max_len) 

# + id="uECocefQKIsI"
print([i for i in sequences[0]])

# + colab={"base_uri": "https://localhost:8080/"} id="ONWuX8VL7X1U" outputId="cf374bd7-b95c-49da-e217-6a1e9d4e18e0"
print("Found", len(train_df.relations.unique()), "relations")

# + id="6nOQ3X5E7m0m"
dictionary = {}
rev_dictionary = {}
for i, rel in enumerate(train_df.relations.unique()):
  #map relation to index
  dictionary[rel] = i
  #map index to relation
  rev_dictionary[i] = rel

# + colab={"base_uri": "https://localhost:8080/"} id="VZOnydCg78vq" outputId="09da1ece-960d-4087-b62b-6b17f55061f9"
print("Length of Relations = ", len(dictionary))
for key, values in dictionary.items():
  print(values, "  ----->   ", key)

# + id="C99aHTby7_mu"
# encoding relations -- (relations to numbers)

encoded_relations = []
for rel in train_df['relations']:
  encoded_relations.append(dictionary[rel])
train_df['encoded_relations'] = encoded_relations

# + colab={"base_uri": "https://localhost:8080/", "height": 293} id="2rVD0hIH8nns" outputId="4512c6a3-15f6-4ec6-e46b-52c8a5f4be14"
train_df.head()

# + id="6EgQO8l88qKg"
y = np_utils.to_categorical(train_df['encoded_relations'], num_classes=len(train_df.encoded_relations.unique()))
X_train, X_test, y_train, y_test = train_test_split(X , y, test_size=0.20, random_state=42)

# + colab={"base_uri": "https://localhost:8080/"} id="bU8RXSl49B7h" outputId="76eeb484-d8a6-49a0-c94e-4b629679c5a7"
embedding_size = 300
model = Sequential()
model.add(Embedding(n_most_common_words, embedding_size, input_length=X.shape[1]))
model.add(Bidirectional(LSTM(128, dropout=0.7, recurrent_dropout=0.7)))
model.add(Dense(y.shape[1], activation='softmax'))
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['acc'])
print(model.summary())

# + colab={"base_uri": "https://localhost:8080/"} id="Be6Lr2OG9E-Q" outputId="1cff89bd-2d71-4df4-b428-188441291253"
model.fit(X_train, y_train, epochs=10, batch_size=128, validation_split=0.2)

# + colab={"base_uri": "https://localhost:8080/"} id="m4pWILWQ9Kii" outputId="07ca74e9-cf51-4e26-daa9-b3461a908114"
prediction_probas = model.predict(X_test)
predictions = [np.argmax(pred) for pred in prediction_probas]
print(predictions[:10])

# + id="I9nV4gS0ICja"
y_test_dup=np.argmax(y_test, axis=1)

# + id="wzlDFQymIHdi"
print(confusion_matrix(y_test_dup, predictions))

# + colab={"base_uri": "https://localhost:8080/"} id="sWpArTNoIKYm" outputId="b717b6ab-9bc3-4fee-f965-c83a4f9529b9"
print(classification_report(y_test_dup, predictions, digits=3))

# + id="exiI0t1rIT6O"
# Testing ------
seq = tokenizer.texts_to_sequences(['<NAME> Founded Apple'])

# + id="A-XDWUSVI-uT"
test_sent = pad_sequences(seq, max_len)

# + id="lDIfuCUJJN6h"
test_sent_pred = model.predict(test_sent)

# + colab={"base_uri": "https://localhost:8080/"} id="gUEym309Jyhj" outputId="9f462e20-0830-4425-801e-de92591c31ef"
# Output -----
print(rev_dictionary[np.argmax(test_sent_pred)])

# + id="zRXMLELOJ8n4"
model.save('../../models/tensorflow_bidirectional_model.h5')

# +
# Read Write File


def write_modelsfile(dictionary):
    '''
    write dictionary to models_list file in models directory
    '''
    dict_file = open("models_list", "wb")
    pickle.dump(dictionary, dict_file)  
    f.close()


def read_modelsfile():
    '''
    returns dictionary containing the links of models
    '''
    L = None
    f = open("models_list", 'rb')
    
    while True:
        try:
            L = pickle.load(f)  
            break    
        except EOFError:
            print("Completed reading details")
    f.close()
    return L
# -



StarcoderdataJupyterScriptsDedupFiltered