# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Semantic Textual Similarity

# !pip install modin

# !pip install modin[dask]

# !which pip

# !conda install pandas=0.25.3

# !export MODIN_ENGINE=ray  # Modin will use Ray

# !export MODIN_ENGINE=dask  # Modin will use Dask

import pandas as pd

 pd.__version__

data = pd.read_csv('Text_Similarity_Dataset.csv')


# +

data.head()
# -

# I wanted to do TF-IDF with synonyms

# word embedding / encoding - Transfer learning

# other approaches - Doc2Vec, Universal Sentence Encoder, transformers (https://huggingface.co/transformers/v2.1.1/)

# cosine-similiarity, or manhatten / euclidean distance to find sentence embeddings that are closest = that are the most similar.

# other good recent approaches ALBERT- https://arxiv.org/pdf/1909.11942v3.pdf

data.text1[2]

# # Doc2Vec

# We have to convert the text1 and text2 to some intermediate representation through some encoding then find the similarity between the encoded vectors

# The approach I have used illustrated in the below image. Image is taken from Andrew Ng's course on Sequence Models

# ![hi](emb.jpeg "Encoding")

# The dataset looks like news articles (just by looking at the beginnig 3 words), so I am going to use [AP-NEWS Pre-Trained model](https://ibm.ent.box.com/s/9ebs3c759qqo1d8i7ed323i6shv2js7e) (a collection of Associated PressEnglish news articles from 2009 to 2015) from [An Empirical Evaluation of doc2vec with Practical Insights into Document Embedding Generation](https://arxiv.org/pdf/1607.05368.pdf)

# Other pre trained corpus like **GoogleNews-vectors-negative** can also be used

#python example to infer document vectors from trained doc2vec model
import gensim.models as g
import codecs
import numpy as np

#parameters
model="apnews_dbow/doc2vec.bin"

#inference hyper-parameters
start_alpha=0.01
infer_epoch=1000

#load model
m = g.Doc2Vec.load(model)

# I am going to keeping stopwords and punctuation

test_docs_1 = [x.strip().split() for x in data.text1]

test_docs_2 = [x.strip().split() for x in data.text2]

# %load_ext Cython

# %load_ext cython

# The cell below takes time to run, I have saved the output in the file **'text1_vec.npy'**, it can be loaded directly

# CPU times: user 28min 22s, sys: 2.93 s, total: 28min 25s <br />
# Wall time: 28min 26s

from multiprocessing import Pool

cores = 4  # number of cores available

# + jupyter={"outputs_hidden": true}
# m.infer_vector?
# -

# %%time
with Pool(cores) as p:
    text1_rep = p.map(m.infer_vector, test_docs_1)

# +
# # %%time
# start_alpha=0.01
# infer_epoch=1000
# text1_rep = [m.infer_vector(d, alpha=start_alpha, steps=infer_epoch) for d in test_docs_1]
# -

text1_rep_np = np.array(text1_rep)

np.save('text1_vec',text1_rep_np)

text1_vec_load = np.load('text1_vec.npy')

np.frombuffer(test)                                  # 459ns !!!

# The cell below takes time to run, I have saved the output in the file **'text2_vec.npy'**, it can be loaded directly

# CPU times: user 27min 44s, sys: 1.43 s, total: 27min 46s </br>
# Wall time: 27min 46s

# %%time
with Pool(cores) as p:
    text2_rep = p.map(m.infer_vector, test_docs_2)

# +
# # %%time
# text2_rep = [m.infer_vector(d, alpha=start_alpha, steps=infer_epoch) for d in test_docs_2]
# -

text2_rep_np = np.array(text2_rep)

np.save('text2_vec',text2_rep_np)

text2_vec_load = np.load('text2_vec.npy')

from sklearn.metrics.pairwise import cosine_similarity

text1_vec_load.shape

# %%time
cos_sim = cosine_similarity(text1_vec_load, text2_vec_load)

from multiprocessing import Process

# %%time
if __name__ == '__main__':
    p = Process(target=cosine_similarity, args=(text1_vec_load, text2_vec_load))
    p.start()
    p.join()

cosine_sim = []

# %%time
for i in range(len(data)):
    cosine_sim.append(cos_sim[i][i])

cosine_sim = np.array(cosine_sim)

maxx = np.argmax([x for x in cosine_sim if x != max(cosine_sim)])

minn = min(cosine_sim)

minn

temp_data = data

maxx



data.text1[maxx]

data.text2[maxx]

# ## Plotting the cosine_sim



# %matplotlib inline
import matplotlib.pyplot as plt

type(data.Unique_ID)

type(np.array(data.Unique_ID))

import seaborn as sns

x = np.array(data.Unique_ID)
y = cosine_sim
plt.scatter(x,y)
plt.title("before tuning")
plt.xlabel("Unique_ID")
plt.ylabel("cosine_similarity")
plt.show()

# # Transfer Learning - Trainig - fine tuning the the apnews_dbow model



# +
import logging

#doc2vec parameters
vector_size = 300
window_size = 15
min_count = 1
sampling_threshold = 1e-5
negative_size = 5
train_epoch = 100
dm = 0 #0 = dbow; 1 = dmpv
worker_count = -1 #number of parallel processes

# +

#pretrained word embeddings
pretrained_emb = 'doc2vec.bin'
# pretrained_emb = "toy_data/pretrained_word_embeddings.txt" #None if use without pretrained embeddings

#input corpus
train_corpus = "my_train_docs.txt"

#output model
saved_path = "my_model.bin"
# -

#enable logging
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)


text1 = ''
i=0
for x in data.text1:
    i+= 1
    text1 += x
    text1 += '\n'

text2 = ''
i=0
for x in data.text2:
    i+= 1
    text2 += x
    text2 += '\n'

text = text1 + text2

with open(train_corpus, mode = 'w') as f:
    f.write(text)

# %%time
#train doc2vec model
docs = g.doc2vec.TaggedLineDocument(train_corpus)

# + jupyter={"outputs_hidden": true}
# %%time
#train doc2vec model
model = g.Doc2Vec(docs, size=vector_size, window=window_size, min_count=min_count, sample=sampling_threshold, workers=worker_count, hs=0, dm=dm, negative=negative_size, dbow_words=1, dm_concat=1, pretrained_emb=pretrained_emb, iter=train_epoch)

# -

#save model
model.save(saved_path)

# ## Loading the tuned model and encoding the paragraphs

#parameters
model="my_model.bin"

#inference hyper-parameters
start_alpha=0.01
infer_epoch=1000

#load model
m2 = g.Doc2Vec.load(model)



# %%time
with Pool(cores) as p:
    my_text1_rep = p.map(m2.infer_vector, test_docs_1)

len(my_text1_rep[1])

my_text1_rep_np = np.array(my_text1_rep)

my_text1_rep_np.shape

# %%time
with Pool(cores) as p:
    my_text2_rep = p.map(m2.infer_vector, test_docs_2)

np.save('after_tuning/tuned_text1_vec.npy',my_text1_rep)

np.save('after_tuning/tuned_text2_vec.npy',my_text2_rep)

my_text1_vec_load = np.load('after_tuning/tuned_text1_vec.npy')

my_text2_vec_load = np.load('after_tuning/tuned_text2_vec.npy')

# %%time
tuned_cos_sim = cosine_similarity(my_text1_vec_load, my_text2_vec_load)

tuned_cosine_sim = []

# %%time
for i in range(len(data)):
    tuned_cosine_sim.append(tuned_cos_sim[i][i])

minn = min(tuned_cosine_sim)

minn

# +
x = np.array(data.Unique_ID)
y = tuned_cosine_sim

plt.scatter(x,y)
plt.title("after tuning")
plt.xlabel("Unique_ID")
plt.ylabel("cosine_similarity")
plt.show()
# -



StarcoderdataJupyterScriptsDedupFiltered