# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python [conda env:py35]
#     language: python
#     name: conda-env-py35-py
# ---

# +
from tensorflow.python.keras.preprocessing.text import Tokenizer
from tensorflow.python.keras.preprocessing import sequence
from tqdm import tqdm

import os
import tempfile
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorboard import summary as summary_lib
import matplotlib.pyplot as plt
# %matplotlib inline
tf.__version__
# -

csv_file = "/Users/i337036/Downloads/train.csv"
df = pd.read_csv(csv_file, encoding='utf-8')
len(df)

df.columns.values

# Remove cell later to train better
df = df.drop_duplicates(subset=['question1'], keep='first')
df = df.drop_duplicates(subset=['question2'], keep='first')
len(df)

q1_list = df.question1.tolist()
q2_list = df.question2.tolist()
print(len(q1_list), len(q2_list))

q1q2_list = ["%s %s" % (txt1, txt2) for txt1, txt2 in zip(q1_list, q2_list)]
print(len(q1q2_list))

labels = df["is_duplicate"].tolist()
print(len(labels))

nb_words = 5000
tokenizer = Tokenizer(num_words=nb_words)

tokenizer.fit_on_texts(q1q2_list)
print(len(tokenizer.word_index))

x_train_variable, y_train = [], []
x_test_variable, y_test = [], []
for idx in tqdm(range(len(q1q2_list))):
    if idx < 200000:
        x_train_variable.append(tokenizer.texts_to_sequences([q1q2_list[idx]])[0])
        y_train.append(labels[idx])
    else:
        x_test_variable.append(tokenizer.texts_to_sequences([q1q2_list[idx]])[0])
        y_test.append(labels[idx])

# +
vocab_size = 5000
sentence_size = 100
embedding_size = 50
model_dir = tempfile.mkdtemp()

y_train = np.array(y_train)
y_test = np.array(y_test)
# we assign the first indices in the vocabulary to special tokens that we use
# for padding, as start token, and for indicating unknown words
pad_id = 0
# -

model_dir

print("Pad sequences (samples x time)")
x_train = sequence.pad_sequences(x_train_variable, maxlen=sentence_size, truncating='post', padding='post', value=pad_id)
x_test = sequence.pad_sequences(x_test_variable, maxlen=sentence_size, truncating='post', padding='post', value=pad_id)

word_index = tokenizer.word_index
word_inverted_index = {v: k for k, v in word_index.items()}

# +
word_inverted_index[pad_id] = '<PAD>'
for i in range(0, 10):
    print(i, word_inverted_index[i])

def index_to_text(indexes):
    return ' '.join([word_inverted_index[i] for i in indexes])

print(index_to_text(x_train_variable[0]))

# +
x_len_train = np.array([min(len(x), sentence_size) for x in x_train_variable])
x_len_test = np.array([min(len(x), sentence_size) for x in x_test_variable])

def parser(x, length, y):
    features = {"x": x, "len": length}
    return features, y

def train_input_fn():
    dataset = tf.data.Dataset.from_tensor_slices((x_train, x_len_train, y_train))
    dataset = dataset.shuffle(buffer_size=len(x_train_variable))
    dataset = dataset.batch(100)
    dataset = dataset.map(parser)
    dataset = dataset.repeat()
    iterator = dataset.make_one_shot_iterator()
    return iterator.get_next()

def eval_input_fn():
    dataset = tf.data.Dataset.from_tensor_slices((x_test, x_len_test, y_test))
    dataset = dataset.batch(100)
    dataset = dataset.map(parser)
    iterator = dataset.make_one_shot_iterator()
    return iterator.get_next()


# -

column = tf.feature_column.categorical_column_with_identity('x', vocab_size)
classifier = tf.estimator.LinearClassifier(feature_columns=[column], model_dir=os.path.join(model_dir, 'bow_sparse'))

all_classifiers = {}
def train_and_evaluate(classifier):
    # Save a reference to the classifier to run predictions later
    all_classifiers[classifier.model_dir] = classifier
    classifier.train(input_fn=train_input_fn, steps=25000)
    eval_results = classifier.evaluate(input_fn=eval_input_fn)
    predictions = np.array([p['logistic'][0] for p in classifier.predict(input_fn=eval_input_fn)])
        
    # Reset the graph to be able to reuse name scopes
    tf.reset_default_graph() 
    # Add a PR summary in addition to the summaries that the classifier writes
    pr = summary_lib.pr_curve('precision_recall', predictions=predictions, labels=y_test.astype(bool), num_thresholds=21)
    with tf.Session() as sess:
        writer = tf.summary.FileWriter(os.path.join(classifier.model_dir, 'eval'), sess.graph)
        writer.add_summary(sess.run(pr), global_step=0)
        writer.close()


train_and_evaluate(classifier)

# +
weights = classifier.get_variable_value('linear/linear_model/x/weights').flatten()
sorted_indexes = np.argsort(weights)
extremes = np.concatenate((sorted_indexes[-8:], sorted_indexes[:8]))
extreme_weights = sorted([(weights[i], word_inverted_index[i]) for i in extremes])

y_pos = np.arange(len(extreme_weights))
plt.bar(y_pos, [pair[0] for pair in extreme_weights], align='center', alpha=0.5)
plt.xticks(y_pos, [pair[1] for pair in extreme_weights], rotation=45, ha='right')
plt.ylabel('Weight')
plt.title('Most significant tokens') 
plt.show()
# -

word_embedding_column = tf.feature_column.embedding_column(column, dimension=embedding_size)
classifier = tf.estimator.DNNClassifier(
    hidden_units=[100],
    feature_columns=[word_embedding_column], 
    model_dir=os.path.join(model_dir, 'bow_embeddings'))
train_and_evaluate(classifier)

with open(os.path.join(model_dir, 'metadata.tsv'), 'w', encoding="utf-8") as f:
    f.write('label\n')
    for index in range(0, vocab_size):
        f.write(word_inverted_index[index] + '\n')

# +
head = tf.contrib.estimator.binary_classification_head()

def cnn_model_fn(features, labels, mode, params):    
    input_layer = tf.contrib.layers.embed_sequence(
        features['x'], vocab_size, embedding_size,
        initializer=params['embedding_initializer'])
    
    training = mode == tf.estimator.ModeKeys.TRAIN
    dropout_emb = tf.layers.dropout(inputs=input_layer, 
                                    rate=0.2, 
                                    training=training)

    conv = tf.layers.conv1d(
        inputs=dropout_emb,
        filters=32,
        kernel_size=3,
        padding="same",
        activation=tf.nn.relu)
    
    # Global Max Pooling
    pool = tf.reduce_max(input_tensor=conv, axis=1)
    
    hidden = tf.layers.dense(inputs=pool, units=250, activation=tf.nn.relu)
    
    dropout_hidden = tf.layers.dropout(inputs=hidden, 
                                       rate=0.2, 
                                       training=training)
    
    logits = tf.layers.dense(inputs=dropout_hidden, units=1)
    
    # This will be None when predicting
    if labels is not None:
        labels = tf.reshape(labels, [-1, 1])
        

    optimizer = tf.train.AdamOptimizer()
    
    def _train_op_fn(loss):
        return optimizer.minimize(
            loss=loss,
            global_step=tf.train.get_global_step())

    return head.create_estimator_spec(
        features=features,
        labels=labels,
        mode=mode,
        logits=logits, 
        train_op_fn=_train_op_fn)
  
params = {'embedding_initializer': tf.random_uniform_initializer(-1.0, 1.0)}
cnn_classifier = tf.estimator.Estimator(model_fn=cnn_model_fn,
                                        model_dir=os.path.join(model_dir, 'cnn'),
                                        params=params)
# -

train_and_evaluate(cnn_classifier)



StarcoderdataJupyterScriptsDedupFiltered