# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="Jbvf2aHA0jNs" colab_type="text"
# # Synthetic Features Engineering Notebook on Google Colab (GPU)
#
# This notebook is modified to work on Google Colab, which provides free GPU / TPU resources for machine learning tasks.
#
# Everyone who has access to this google drive folder should have access to this notebook and should be edit it directly on Google Colab.
#

# + [markdown] id="1F5WY5hw2BDK" colab_type="text"
# ## README and Developer Guide
#
# We will write and modify code DIRECTLY on **Google Colab**, which is not automatically synced with the version on our github repository. Make sure that you **DOWNLOAD** this notebook after edit and upload it to our **github repository** for a more consistent version control.
#
# Make sure you add the Paradigm folder to your drive before runnning the code. To do so right click the folder and select **Add to my drive** from the dropdown menu.
#
# Because of the nature of Google Drive, it is very hard to edit the same ipython file on the same time on Colab. If someone is editing the colab file at the same time as you, you will need to **restart** the runtime, so be mindful of that if someone is editing the notebook at the same time as you.
#
# We are fairly new to Google Colab and thus feel free to add anything to the README section if you come across anything that is important when coding!
#
# This notebook will be geared to use **GPU** to help accelerate the training process. Please make sure that when you run this notebook the backend if using **GPU**. To do so click 'Runtime' then select 'Change runtime type' and make sure that the hardware accelerator is set to GPU. If you are interested in using **TPU** to accelerate the training process even **FASTER**, please create another notebook to run the TPU-Compatible model.

# + [markdown] id="G4cIzHHy39-M" colab_type="text"
# ## (0) SETUP
#
# 1.   Grant Google Colab access to Google Drive
# 2.   Import libraries and tools
#
#
#
#
#
#

# + id="2CrtVDmg0kIw" colab_type="code" outputId="b57dac5f-dc0d-4e11-f010-08e53492b260" executionInfo={"status": "ok", "timestamp": 1554922565592, "user_tz": 420, "elapsed": 21368, "user": {"displayName": "<NAME>", "photoUrl": "https://lh4.googleusercontent.com/-pkp40ccE7So/AAAAAAAAAAI/AAAAAAAAAU4/Upp1QcV6fHs/s64/photo.jpg", "userId": "16137932526864003348"}} colab={"base_uri": "https://localhost:8080/", "height": 122}
# Getting access to the dataset and the Python files on Google Drive.
# You will have to give permission and sign in to your Google account.

from google.colab import drive
drive.mount('/content/gdrive')
root_folder = "/content/gdrive/My Drive/Paradigm/Paradigm (Spr 19) - Team 2/Colab-code/"

# + id="U8hKmXJyZSwN" colab_type="code" outputId="d0230606-611c-450b-b993-d422c34c7944" executionInfo={"status": "ok", "timestamp": 1554922571556, "user_tz": 420, "elapsed": 27266, "user": {"displayName": "<NAME>", "photoUrl": "https://lh4.googleusercontent.com/-pkp40ccE7So/AAAAAAAAAAI/AAAAAAAAAU4/Upp1QcV6fHs/s64/photo.jpg", "userId": "16137932526864003348"}} colab={"base_uri": "https://localhost:8080/", "height": 190}
# install segtok
# !pip install segtok

# + id="4wgY1xMi43ms" colab_type="code" colab={}
# general imports
from __future__ import absolute_import, division, print_function

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import collections
import os
import string
import time
from segtok import tokenizer
from collections import Counter
import json
import sys

# + id="KK9eQkWi5Poc" colab_type="code" outputId="4f615ba1-8491-4c04-b7f3-534e9bdc96e3" executionInfo={"status": "ok", "timestamp": 1554922573967, "user_tz": 420, "elapsed": 29625, "user": {"displayName": "<NAME>", "photoUrl": "https://lh4.googleusercontent.com/-pkp40ccE7So/AAAAAAAAAAI/AAAAAAAAAU4/Upp1QcV6fHs/s64/photo.jpg", "userId": "16137932526864003348"}} colab={"base_uri": "https://localhost:8080/", "height": 34}
# machine learning libraries imports
import keras
import sklearn
import tensorflow as tf

from keras import backend as K
from keras import layers
from keras.models import Model, Sequential
from keras.layers import Dense, Embedding, Input, Lambda, LSTM, Masking, RepeatVector, TimeDistributed
from keras.preprocessing.text import Tokenizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split

# + [markdown] id="XkaR9Zy2JRa_" colab_type="text"
# ## (1) EDA
#
#
# 1.   Import data 
# 2.   Analysize data
# 3.   Clean data
# 4.  Graph relationships
#
# **NOTE**: The dataset we have is not very big (around 7000 cyptocurrency related articles). If time permitted we can scrap older news article from [bitCoinTalk Thread](https://bitcointalk.org/index.php?board=77.0)
#

# + [markdown] id="RdejphAAVlW4" colab_type="text"
# ### (1.1) Import data from the data folder

# + id="0QYxDh2l5T5m" colab_type="code" outputId="087840c5-a28c-40d6-8498-134e32fc6396" executionInfo={"status": "ok", "timestamp": 1554922575112, "user_tz": 420, "elapsed": 30671, "user": {"displayName": "<NAME>", "photoUrl": "https://lh4.googleusercontent.com/-pkp40ccE7So/AAAAAAAAAAI/AAAAAAAAAU4/Upp1QcV6fHs/s64/photo.jpg", "userId": "16137932526864003348"}} colab={"base_uri": "https://localhost:8080/", "height": 51}
# get a list of data we have

data_folder = root_folder + 'data/'
print("We have gathered the following datasets")
print(os.listdir(data_folder))


# + id="_mqvydQbLTEZ" colab_type="code" colab={}
# importing data from csv to dataframe
news_score_df = pd.read_csv(data_folder + 'news_score.csv')
raw_data_df = pd.read_csv(data_folder + 'rawData_test1009.csv')
cleaned_author_df = pd.read_csv(data_folder + '1119_cleaned_author_articles.csv')
onehot_df = pd.read_csv(data_folder + 'data_onehot.csv')

# + [markdown] id="cmRmIt1YNwYj" colab_type="text"
# ### (1.2) EDA on News Score Dataframe

# + id="pGxsSKlXNMRr" colab_type="code" outputId="d6504189-7c17-41b1-9668-403575510186" executionInfo={"status": "ok", "timestamp": 1554655011967, "user_tz": 420, "elapsed": 761, "user": {"displayName": "<NAME>", "photoUrl": "https://lh4.googleusercontent.com/-pkp40ccE7So/AAAAAAAAAAI/AAAAAAAAAU4/Upp1QcV6fHs/s64/photo.jpg", "userId": "16137932526864003348"}} colab={"base_uri": "https://localhost:8080/", "height": 576}
print("There are " + str(len(news_score_df)) + " entries in this dataframe.")
sum_of_nans = sum(len(news_score_df) - news_score_df.count())
print("There are " + str(sum_of_nans) + " Nan values in the dataframe.")

# take a look at the news scores dataframe
news_score_df.head()

# + id="d13QewRKQrz4" colab_type="code" outputId="2806a48b-cacf-4343-e9fb-8df6690b74eb" executionInfo={"status": "ok", "timestamp": 1554655012369, "user_tz": 420, "elapsed": 602, "user": {"displayName": "<NAME>", "photoUrl": "https://lh4.googleusercontent.com/-pkp40ccE7So/AAAAAAAAAAI/AAAAAAAAAU4/Upp1QcV6fHs/s64/photo.jpg", "userId": "16137932526864003348"}} colab={"base_uri": "https://localhost:8080/", "height": 459}
# find out which column has NAN values
news_score_df.isna().sum()

# + id="7UukGv2EORHW" colab_type="code" outputId="5691153f-b998-495e-88fe-01328838d176" executionInfo={"status": "ok", "timestamp": 1554655016365, "user_tz": 420, "elapsed": 703, "user": {"displayName": "<NAME>", "photoUrl": "https://lh4.googleusercontent.com/-pkp40ccE7So/AAAAAAAAAAI/AAAAAAAAAU4/Upp1QcV6fHs/s64/photo.jpg", "userId": "16137932526864003348"}} colab={"base_uri": "https://localhost:8080/", "height": 119}
# get the list of news score dataframe columns
print(news_score_df.columns)

# + id="dfR5QNEmOlnP" colab_type="code" outputId="7c78dace-3684-4184-e3eb-555387622cf8" executionInfo={"status": "ok", "timestamp": 1554655016954, "user_tz": 420, "elapsed": 807, "user": {"displayName": "<NAME>", "photoUrl": "https://lh4.googleusercontent.com/-pkp40ccE7So/AAAAAAAAAAI/AAAAAAAAAU4/Upp1QcV6fHs/s64/photo.jpg", "userId": "16137932526864003348"}} colab={"base_uri": "https://localhost:8080/", "height": 334}
# describe the dataframe
news_score_df.describe()

# + [markdown] id="xYYA6GfFN1mb" colab_type="text"
# ### (1.3) EDA on Raw Data Dataframe

# + id="M3M26sfjQ_nT" colab_type="code" outputId="d21635d9-b8c3-45d8-89dd-2572df02b6dc" executionInfo={"status": "ok", "timestamp": 1554655021326, "user_tz": 420, "elapsed": 883, "user": {"displayName": "<NAME>", "photoUrl": "https://lh4.googleusercontent.com/-pkp40ccE7So/AAAAAAAAAAI/AAAAAAAAAU4/Upp1QcV6fHs/s64/photo.jpg", "userId": "16137932526864003348"}} colab={"base_uri": "https://localhost:8080/", "height": 323}
print("There are " + str(len(raw_data_df)) + " entries in this dataframe.")
sum_of_nans = sum(len(raw_data_df) - raw_data_df.count())
print("There are " + str(sum_of_nans) + " Nan values in the dataframe.")

# take a look at the news scores dataframe
raw_data_df.head()

# + [markdown] id="ydsc-4NUN6jA" colab_type="text"
# ### (1.4) EDA on Cleaned Author Dataframe

# + id="y6vIkwK-N8sa" colab_type="code" outputId="a83cd2a8-f371-4a32-9040-af776169ac88" executionInfo={"status": "ok", "timestamp": 1554655022608, "user_tz": 420, "elapsed": 1175, "user": {"displayName": "SHUN LIN", "photoUrl": "https://lh4.googleusercontent.com/-pkp40ccE7So/AAAAAAAAAAI/AAAAAAAAAU4/Upp1QcV6fHs/s64/photo.jpg", "userId": "16137932526864003348"}} colab={"base_uri": "https://localhost:8080/", "height": 559}
print("There are " + str(len(cleaned_author_df)) + " entries in this dataframe.")
sum_of_nans = sum(len(cleaned_author_df) - cleaned_author_df.count())
print("There are " + str(sum_of_nans) + " Nan values in the dataframe.")

# take a look at the news scores dataframe
cleaned_author_df.head()

# + [markdown] id="Z4nsUEh9OIUV" colab_type="text"
# ### EDA on One Hot Dataframe

# + id="H3bipu32OLiA" colab_type="code" outputId="dd1200a0-9e39-4a27-9298-9a5c6f5d1b34" executionInfo={"status": "ok", "timestamp": 1554655023948, "user_tz": 420, "elapsed": 1395, "user": {"displayName": "<NAME>", "photoUrl": "https://lh4.googleusercontent.com/-pkp40ccE7So/AAAAAAAAAAI/AAAAAAAAAU4/Upp1QcV6fHs/s64/photo.jpg", "userId": "16137932526864003348"}} colab={"base_uri": "https://localhost:8080/", "height": 576}
print("There are " + str(len(onehot_df)) + " entries in this dataframe.")
sum_of_nans = sum(len(onehot_df) - onehot_df.count())
print("There are " + str(sum_of_nans) + " Nan values in the dataframe.")

# take a look at the news scores dataframe
onehot_df.head()


# + [markdown] id="dEvVjZn3SkXP" colab_type="text"
# ## (2) Language Models
#
# We will try to build a language model for our news score dataset to extract additional information.
#
# **NOTE**: We will be trying to build different language models with different architectures. We will also build models using tensorflow and keras for comparsion.

# + [markdown] id="ylPt5Z28Xj5H" colab_type="text"
# ### (2.1) Preprocessing our data

# + id="3J_QxrpFS2ks" colab_type="code" colab={}
# helper methods

def numerize_sequence(tokenized):
    return [w2i.get(w, unkI) for w in tokenized]
def pad_sequence(numerized, pad_index, to_length):
    pad = numerized[:to_length]
    padded = pad + [pad_index] * (to_length - len(pad))
    mask = [w != pad_index for w in padded]
    return padded, mask


# + id="mMjEz58vV7P0" colab_type="code" colab={}
dataset_df = news_score_df[["title"]]
dataset = dataset_df.to_dict('records')

# + id="ofsrpPapYMuq" colab_type="code" outputId="96fe8951-41ff-45bc-cdea-2a0d56918e2f" executionInfo={"status": "ok", "timestamp": 1554922594208, "user_tz": 420, "elapsed": 1570, "user": {"displayName": "<NAME>", "photoUrl": "https://lh4.googleusercontent.com/-pkp40ccE7So/AAAAAAAAAAI/AAAAAAAAAU4/Upp1QcV6fHs/s64/photo.jpg", "userId": "16137932526864003348"}} colab={"base_uri": "https://localhost:8080/", "height": 34}
input_length = 0
for a in dataset:
    tokenized_title = tokenizer.word_tokenizer(a['title'].lower())
    input_length = max(input_length, len(tokenized_title))
    a['tokenized'] = tokenized_title
print(input_length)

# + id="iecSKIzrYNYp" colab_type="code" colab={}
word_counts = Counter()
for a in dataset:
    word_counts.update(a['tokenized'])

# + id="Ze5LV5ijZgi3" colab_type="code" outputId="69692f93-9edc-419f-af9a-eec950b9cae6" executionInfo={"status": "ok", "timestamp": 1554922596105, "user_tz": 420, "elapsed": 897, "user": {"displayName": "<NAME>", "photoUrl": "https://lh4.googleusercontent.com/-pkp40ccE7So/AAAAAAAAAAI/AAAAAAAAAU4/Upp1QcV6fHs/s64/photo.jpg", "userId": "16137932526864003348"}} colab={"base_uri": "https://localhost:8080/", "height": 34}
# Creating the vocab
vocab_size = len(word_counts)
special_words = ["<START>", "UNK", "PAD"]
vocabulary = special_words + [w for w, c in word_counts.most_common(vocab_size-len(special_words))]
w2i = {w: i for i, w in enumerate(vocabulary)}

# Numerizing and padding
unkI, padI, startI = w2i['UNK'], w2i['PAD'], w2i['<START>']

for a in dataset:
    a['numerized'] = numerize_sequence(a['tokenized']) # Change words to IDs
    a['numerized'], a['mask'] = pad_sequence(a['numerized'], padI, input_length) # Append appropriate PAD tokens
    
# Compute fraction of words that are UNK:
word_counters = Counter([w for a in dataset for w in a['title'] if w != padI])

print("Fraction of UNK words:", float(word_counters[unkI]) / sum(word_counters.values()))

# + id="VrAXaUUYZ0WZ" colab_type="code" outputId="0c1ba74d-dd8c-437b-9964-4f930944d5cd" executionInfo={"status": "ok", "timestamp": 1554922600427, "user_tz": 420, "elapsed": 548, "user": {"displayName": "<NAME>", "photoUrl": "https://lh4.googleusercontent.com/-pkp40ccE7So/AAAAAAAAAAI/AAAAAAAAAU4/Upp1QcV6fHs/s64/photo.jpg", "userId": "16137932526864003348"}} colab={"base_uri": "https://localhost:8080/", "height": 68}
vocab_size = len(vocabulary)
input_length = len(dataset[0]['numerized']) # The length of the first element in the dataset, they are all of the same length

d_train, d_valid = train_test_split(dataset, test_size=0.01, random_state=42)

print("Vocabulary Size:", vocab_size)
print("Number of training samples:",len(d_train))
print("Number of validation samples:",len(d_valid))


# + id="vl-epkKRaGmB" colab_type="code" outputId="af7ddb8c-bbf7-4ac9-e265-494c86104ebe" executionInfo={"status": "ok", "timestamp": 1554655071961, "user_tz": 420, "elapsed": 667, "user": {"displayName": "<NAME>", "photoUrl": "https://lh4.googleusercontent.com/-pkp40ccE7So/AAAAAAAAAAI/AAAAAAAAAU4/Upp1QcV6fHs/s64/photo.jpg", "userId": "16137932526864003348"}} colab={"base_uri": "https://localhost:8080/", "height": 71}
def numerized2text(numerized):    
    words = [vocabulary[int(num)] for num in numerized]
    converted_string = ' '.join(words)
    return converted_string

entry = d_train[100]
print("Reversing the numerized: "+numerized2text(entry['numerized']))
print("From the `title` entry: "+ entry['title'])


# + [markdown] id="FoCwB-9xadBx" colab_type="text"
# ### (2.2) Building Model

# + id="P46hY9_PaQ7S" colab_type="code" colab={}
def build_batch(dataset, batch_size):
    
    # randomize the indices we want to get the batch of
    indices = list(np.random.randint(0, len(dataset), size=batch_size))
    
    # indice into the batch
    batch = [dataset[i] for i in indices]
    
    # Get the raw numerized for this input
    batch_numerized = np.asarray([db_element["numerized"] for db_element in batch])

    # Create an array of start_index that will be concatenated at position 1 for input
    start_tokens = np.zeros((batch_size, 1))
    batch_input = np.concatenate((start_tokens, batch_numerized), axis=1)

    # Remove the last word from each element in the batch to "shift" input
    batch_input = batch_input[:, :-1]
    
    # The target should be the un-shifted numerized input
    batch_target = batch_numerized

    # The target-mask is a 0 or 1 filter to note which tokens are
    # padding or not, to give the loss, so the model doesn't get rewarded for
    # predicting PAD tokens.
    batch_target_mask = np.array([a['mask'] for a in batch])
        
    return batch_input, batch_target, batch_target_mask


# + id="_ux4FZaeaw8P" colab_type="code" colab={}
# Using a basic RNN/LSTM for Language modeling
class LanguageModel():
    def __init__(self, input_length, vocab_size, rnn_size, learning_rate=1e-4):
        self.input_num = tf.placeholder(tf.int32, shape=[None, input_length])
        self.targets = tf.placeholder(tf.int32, shape=[None, input_length])
        self.targets_mask = tf.placeholder(tf.bool, shape=[None, input_length])
        self.embedding = tf.Variable(tf.random_uniform([vocab_size, rnn_size], -1.0, 1.0))
        input_emb = tf.nn.embedding_lookup(self.embedding, self.input_num)
        lm_cell = tf.nn.rnn_cell.LSTMCell(rnn_size)
        outputs, states = tf.nn.dynamic_rnn(lm_cell, input_emb, dtype=tf.float32)
        self.output_logits = tf.layers.dense(inputs=outputs, units=vocab_size)
        weights = tf.cast(self.targets_mask, tf.float32)
        self.loss = tf.losses.sparse_softmax_cross_entropy(labels=self.targets,logits=self.output_logits, weights=weights)
        optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate, name='Adam')     
        self.global_step = tf.train.get_or_create_global_step()
        self.train_op = optimizer.minimize(self.loss, global_step=self.global_step)
        self.saver = tf.train.Saver()


# + [markdown] id="48Cy77pXbD1m" colab_type="text"
# ### (2.3) Create Model

# + id="sCamFzWFa1VB" colab_type="code" outputId="abc03c34-a9e4-4187-f1c4-24663fe95a5b" executionInfo={"status": "ok", "timestamp": 1554655077570, "user_tz": 420, "elapsed": 1909, "user": {"displayName": "<NAME>", "photoUrl": "https://lh4.googleusercontent.com/-pkp40ccE7So/AAAAAAAAAAI/AAAAAAAAAU4/Upp1QcV6fHs/s64/photo.jpg", "userId": "16137932526864003348"}} colab={"base_uri": "https://localhost:8080/", "height": 343}
tf.reset_default_graph()
model = LanguageModel(input_length=input_length, vocab_size=vocab_size, rnn_size=256, learning_rate=1e-4)

# + [markdown] id="-s8bZMkkbTcU" colab_type="text"
# ### (2.4) Train Model

# + id="FtfYzgENbWBZ" colab_type="code" outputId="1f5a5616-8b4e-4fa1-af04-bdee4f340b63" colab={"base_uri": "https://localhost:8080/", "height": 37893}
# DO NOT RUN THIS BLOCK IF YOU DON'T WANT TO TRAIN THE NETWORK

experiment = root_folder+"models/tf_language_model"
plot_info_path = root_folder+"plots/tf_language_model.csv"

plot_info = pd.DataFrame(columns=['training_err', 'validation_err'])

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    
    # Here is how you restore the weights previously saved
    # model.saver.restore(sess, experiment)
    # also need to restore plot_info from plot_info_path
    
    epoch = 20000000
    batch_size = 64
    num_iter = epoch * len(d_train) // batch_size
    print("Total number of iterations is: " + str(num_iter))
    
    eval_input, eval_target, eval_target_mask = build_batch(d_valid, 50)
    feed = {model.input_num: eval_input, model.targets: eval_target, model.targets_mask: eval_target_mask}
    eval_loss = sess.run(model.loss, feed_dict=feed)
    print("Evaluation set loss: ", eval_loss)
        
    for i in range(num_iter):
        # Here is how you obtain a batch:
        batch_input, batch_target, batch_target_mask = build_batch(d_train, batch_size)
        # Map the values to each tensor in a `feed_dict`
        feed = {model.input_num: batch_input, model.targets: batch_target, model.targets_mask: batch_target_mask}

        # Obtain a single value of the loss for that batch.
        # # !IMPORTANT! Don't forget to include the train_op to when using a batch from the training dataset
        # (d_train)
        # # !MORE IMPORTANT! Don't use the train_op if you evaluate the loss on the validation set,
        # Otherwise, your network will overfit on your validation dataset.

        step, train_loss, _ = sess.run([model.global_step, model.loss, model.train_op], feed_dict=feed)

        # record info for graphs every 20 steps
        if i % 20 == 0 and i % 200 != 0:
          eval_input, eval_target, eval_target_mask = build_batch(d_valid, 50)
          feed = {model.input_num: eval_input, model.targets: eval_target, model.targets_mask: eval_target_mask}
          eval_loss_steps = sess.run(model.loss, feed_dict=feed)
          row = {'training_err': train_loss, 'validation_err': eval_loss_steps}
          plot_info.loc[len(plot_info)] = row
          
        # save weights info every 200 steps
        if i % 200 == 0:
            print("step: " + str(i))
            print("train_loss: " + str(train_loss))
            eval_input, eval_target, eval_target_mask = build_batch(d_valid, 50)
            feed = {model.input_num: eval_input, model.targets: eval_target, model.targets_mask: eval_target_mask}
            eval_loss_steps = sess.run(model.loss, feed_dict=feed)
            # if (eval_loss_steps < eval_loss):
            #  print("eval_loss decreases!")
            eval_loss = eval_loss_steps
            print("Evaluation set loss: ", eval_loss)
            print("saving plot info so far ....")
            plot_info.to_csv(plot_info_path, index=False)
            print("saving plot info so far completed ....")
            print("saving model weights ....")
            model.saver.save(sess, experiment)
            print("saving model weights completed ....")
            # else:
            #  print("eval_loss didn't decrease.")
            #  print("half learning rate, make another model, reset to previous checkpoint")
            #  # learning_rate /= 2
            #  # model = LanguageModel(input_length=input_length, vocab_size=vocab_size, rnn_size=256*4, learning_rate=learning_rate)
            #  model.saver.restore(sess, experiment)
    
    # Here is how you save the model weights
    model.saver.save(sess, experiment)
    
    # Here is how you restore the weights previously saved
    model.saver.restore(sess, experiment)

# + [markdown] id="0cf96KxecEGV" colab_type="text"
# ### (2.4) Evaluate Models
#
# We will use different methods to evaluate our models
#
# 0. Plot Training and Validation error over epochs
# 1.   How our model generate new titles
# 2.   How our model detects unlikely titles
# 3.   How the embedding perform on our data
#
#

# + [markdown] id="epGLYb4xiZcQ" colab_type="text"
# #### (2.4.0) Plots
#

# + id="f3ymyrZMijK4" colab_type="code" outputId="1c5406d6-7caf-435d-b882-cebedbff5efd" executionInfo={"status": "ok", "timestamp": 1554655087104, "user_tz": 420, "elapsed": 836, "user": {"displayName": "SHUN LIN", "photoUrl": "https://lh4.googleusercontent.com/-pkp40ccE7So/AAAAAAAAAAI/AAAAAAAAAU4/Upp1QcV6fHs/s64/photo.jpg", "userId": "16137932526864003348"}} colab={"base_uri": "https://localhost:8080/", "height": 410}
plot_info_path = root_folder+"plots/tf_language_model.csv"
plot_df = pd.read_csv(plot_info_path)
plot_df_per_epoch = plot_df.groupby(np.arange(len(plot_df))//50).mean()
train_error_per_epoch = plot_df_per_epoch['training_err']
validation_error_per_epoch = plot_df_per_epoch['validation_err']
print(len(plot_df))
f, ax = plt.subplots()
ax.plot(train_error_per_epoch, 'o-',c='r')
ax.plot(validation_error_per_epoch, 'o-',c='g')
# Plot legend and use the best location automatically: loc = 0.
ax.legend(['Train loss', 'Validation loss'], loc = 0)
ax.set_title('Training/Validation loss per Epoch')
ax.set_xlabel('Epoch')
ax.set_ylabel('Error') 
plt.plot()

# + [markdown] id="z6QP9iHRcNlM" colab_type="text"
# #### (2.4.1) Generating New Titles

# + id="_wKS9khlbopZ" colab_type="code" outputId="6cfcc8c3-57ed-4bd2-9a77-715fa7a0ebb0" executionInfo={"status": "ok", "timestamp": 1554655095594, "user_tz": 420, "elapsed": 7084, "user": {"displayName": "SHUN LIN", "photoUrl": "https://lh4.googleusercontent.com/-pkp40ccE7So/AAAAAAAAAAI/AAAAAAAAAU4/Upp1QcV6fHs/s64/photo.jpg", "userId": "16137932526864003348"}} colab={"base_uri": "https://localhost:8080/", "height": 258}
model_file = root_folder+"models/tf_language_model"

with tf.Session() as sess:
    model.saver.restore(sess, model_file)

    # Here are some headline starters.
    # They're all about tech companies, because
    # That is what is in our dataset
    headline_starters = ["bitcoin price", "today", "big"]
    
    for headline_starter in headline_starters:
        print("===================")
        print("Generating titles starting with: "+headline_starter)
        
        tokenized = tokenizer.word_tokenizer(headline_starter)
        current_build = [startI] + numerize_sequence(tokenized)

        while len(current_build) < input_length:
            current_padded = current_build[:input_length] + [padI] * (input_length - len(current_build))
            current_padded = np.array([current_padded])

            feed = {model.input_num: current_padded}
            logits = sess.run(model.output_logits, feed_dict=feed)

            last_index = len(current_build) - 1
            last_logits = logits[0][last_index]
            
            current_build.append(np.argmax(last_logits))
        
        produced_sentence = numerized2text(current_build)
        print(produced_sentence)

# + [markdown] id="TBnVgDvEc9cY" colab_type="text"
# #### (2.4.2) Fake/Unlikely News Headline Detection
#
# Lower loss means the headline is more likely.

# + id="NViFkWxoc3AJ" colab_type="code" outputId="9989d921-f92a-42a2-cdb8-c28b1324f42a" executionInfo={"status": "ok", "timestamp": 1554655095913, "user_tz": 420, "elapsed": 5213, "user": {"displayName": "<NAME>", "photoUrl": "https://lh4.googleusercontent.com/-pkp40ccE7So/AAAAAAAAAAI/AAAAAAAAAU4/Upp1QcV6fHs/s64/photo.jpg", "userId": "16137932526864003348"}} colab={"base_uri": "https://localhost:8080/", "height": 207}
headline1 = "Bitcoin price crashes"
headline2 = "Bitcoin price reach a new high"
headline3 = "bitcoin is legal in all countries"

headlines = [headline1, headline2, headline3]

with tf.Session() as sess:
    model.saver.restore(sess, model_file)

    for headline in headlines:
        headline = headline.lower()

        tokenized = tokenizer.word_tokenizer(headline)
        numerized = numerize_sequence(tokenized)
        unkI, padI, startI = w2i['UNK'], w2i['PAD'], w2i['<START>']
        
        padded, mask = pad_sequence(numerized, padI, input_length)
        
        hl_element = {}
        hl_element['tokenized'] = tokenized
        hl_element['numerized'] = padded
        hl_element['mask'] = mask
        d_hl = [hl_element]
        hl_input, hl_target, hl_target_mask = build_batch(d_hl, 1)
        feed = {model.input_num: hl_input, model.targets: hl_target, model.targets_mask: hl_target_mask}
        loss = sess.run([model.loss], feed_dict=feed)
        print("----------------------------------------")
        print("Headline:",headline)
        print("Loss of the headline:", loss)

# + id="heAfajU4e_ii" colab_type="code" colab={}


StarcoderdataJupyterScriptsDedupFiltered