# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/antonpolishko/task-ties/blob/master/CoronaWhyTopicModel_Bio.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + [markdown] id="enYLtsWu9ShD" colab_type="text"
# #Common API

# + id="-PiQMdOsS55I" colab_type="code" colab={}


# + id="c-s6bow7kWWg" colab_type="code" colab={}
import string
import numpy as np
import pandas as pd

# + [markdown] id="Olubw13U9I1N" colab_type="text"
# #Connect to Google Drive

# + id="BXGGqt9_9g2v" colab_type="code" colab={}
from pathlib import Path, PurePath
from google.colab import drive


# + id="hQxDXQoIjJqR" colab_type="code" colab={}
# set data paths, this requires local drive to have a folder calld "COVID-19" with the metadata.csv file
# returns a string to the local path setup
def setup_local_data():
  
  drive.mount('/content/drive')
  drive_path=PurePath('/content/drive/My Drive')
  input_dir = drive_path/'COVID-19'
  print(list(Path(input_dir).glob('*')))
  return input_dir


# + id="TMd3hEbLj7rs" colab_type="code" colab={}
setup_local_data()


# + [markdown] id="pwdWn9XQ9NcW" colab_type="text"
# #Unzip JSON

# + id="T4Od_SRKj-Rm" colab_type="code" colab={}
def unzip():
    from zipfile import ZipFile
    
    zf = ZipFile('/content/drive/My Drive/COVID-19/TopicModel/covid_data_full_v5.json.zip', 'r')
    zf.extractall('/content/drive/My Drive/COVID-19/')
    zf.close()


# + id="PpYvUdSyk1Q7" colab_type="code" colab={}
#unzip()

# + id="xxJNAEnZk2qZ" colab_type="code" colab={}
def read_json_to_pandas():
   
    pd_json=pd.read_json (r'/content/drive/My Drive/COVID-19/TopicModel/covid_data_full_v5.json')
    return pd_json


# + id="LBRYeINgmIlQ" colab_type="code" outputId="ec74d1ce-6615-4a10-be80-e8924671dafc" colab={"base_uri": "https://localhost:8080/", "height": 589}
All_json_docs=read_json_to_pandas()
All_json_docs

# + id="eOr65HflmKLs" colab_type="code" outputId="5fd64b66-aee8-47bc-f303-0faa2f9de0b0" colab={"base_uri": "https://localhost:8080/", "height": 55}
All_json_docs['text'][33372]

# + id="STUfY7LnmY4o" colab_type="code" outputId="67b93527-b364-4113-96dd-51434e4dfd58" colab={"base_uri": "https://localhost:8080/", "height": 55}
All_json_docs['abstract'][33372]


# + id="hOcbd6pXmtjT" colab_type="code" colab={}
def count_words(sent):
 
   num_words = len(sent.split()) 
   return num_words


# + id="JGg_KChNnS_L" colab_type="code" outputId="4beffcb8-1236-4c02-b439-a98090ad46c9" colab={"base_uri": "https://localhost:8080/", "height": 35}
count_words(All_json_docs['abstract'][33372])

# + id="arwmmg41nVID" colab_type="code" outputId="5889ebae-fa70-4765-e057-9e1d1f9cbca0" colab={"base_uri": "https://localhost:8080/", "height": 35}
count_words(All_json_docs['text'][33372])

# + [markdown] id="77qMMBCTrQwR" colab_type="text"
# #Preprocessing and cleaned

# + id="HNQ_b4opp0XD" colab_type="code" outputId="4e77198f-0bdc-4967-dee6-2a1a57dcdec5" colab={"base_uri": "https://localhost:8080/", "height": 89}
import nltk
nltk.download('stopwords')
nltk.download('punkt')
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize


# + id="eGu-SSqHndDE" colab_type="code" colab={}
def convert_lower_case(data):
    import numpy as np
    return np.char.lower(data)


# + id="pwqUJjkAp_K6" colab_type="code" colab={}
def remove_stop_words(data):
    stop_words = stopwords.words('english')
    words = word_tokenize(str(data))
    new_text = ""
    for w in words:
        if w not in stop_words and len(w) > 1:
            new_text = new_text + " " + w
    return new_text


# + id="5aR0FOH8p_hZ" colab_type="code" colab={}
def remove_punctuation(data):
    symbols = "!\"#$%&()*+-./:;<=>?@[\]^_`{|}~\n"
    for i in range(len(symbols)):
        data = np.char.replace(data, symbols[i], ' ')
        data = np.char.replace(data, "  ", " ")
    data = np.char.replace(data, ',', '')
    return data


# + id="QGmR1QWRqCLI" colab_type="code" colab={}
def remove_apostrophe(data):
    return np.char.replace(data, "'", "")


# + id="d9SsskF2qFh4" colab_type="code" colab={}
def remove_numbers(data):
    from string import digits
    remove_digits = str.maketrans('', '', digits)
    res = data.translate(remove_digits)
    return res


# + id="iLmJmcwp0Rql" colab_type="code" colab={}
def remove_symboles(data):
    import re
    regex = re.compile('[^ s\ a-zA-Z]')#,'[\t\n\r\f\v]')
    data_no_sympoles=regex.sub('',str(data))
    #print(type(data_no_sympoles))
    return data_no_sympoles


# + id="nLibXUm4Cw6w" colab_type="code" outputId="63a96ddd-2993-444a-9bbd-9a8c2b05eeac" colab={"base_uri": "https://localhost:8080/", "height": 55}
remove_symboles(All_json_docs['text'][33372])


# + id="5qGZGtmkqw_c" colab_type="code" colab={}
def preprocess(data):
    
    data = convert_lower_case(data)
    data = remove_punctuation(data) #remove comma seperately
    data = remove_apostrophe(data)
    data = remove_stop_words(data)
    data = remove_numbers(data)
    data = remove_punctuation(data)
    data = remove_symboles(data)
   
    
    return data

# + id="q1bCqt0wq1OM" colab_type="code" colab={}
dete_cleaned=preprocess(All_json_docs['text'][33372])

# + id="QZ7yI3JWCaoH" colab_type="code" outputId="e80c81db-d4ec-4712-80fe-ce25da749d10" colab={"base_uri": "https://localhost:8080/", "height": 55}
dete_cleaned


# + id="0QleE3aJrYyp" colab_type="code" colab={}
#len(dete_cleaned)
#dete_cleaned.size

# + id="G_qR7V_68Fww" colab_type="code" colab={}
def cleaned_COVID19(All_json_docs):
    All_json_docs['cleaned_text'] = All_json_docs['text'].apply(preprocess)
    covid_dataset_dropnan=All_json_docs.dropna()
    All_json_docs.to_csv(r'/content/drive/My Drive/COVID-19/TopicModel/covid_data_full_v5_cleaned.csv',index=False,sep=',',header=True)
    covid_dataset_dropnan.to_csv(r'/content/drive/My Drive/COVID-19/TopicModel/covid_data_full_v5_cleaned_dropnan.csv',index=False,sep=',',header=True)
    return All_json_docs

# + id="NQ6PkhlfsrMI" colab_type="code" colab={}


# + id="RM6u0CkAt5Ns" colab_type="code" colab={}
cleaned_COVID19(All_json_docs)


# + [markdown] id="dLpkZABp9rHi" colab_type="text"
# #Topic Modelling

# + [markdown] id="QGNncyzBLBA6" colab_type="text"
# ##Subset

# + id="kXBNeguc9yne" colab_type="code" colab={}
def read_covid_dataset(columns,n):
    data = pd.read_csv("/content/drive/My Drive/COVID-19/TopicModel/covid_data_full_v5_cleaned_dropnan.csv",usecols= columns, nrows=n,index_col=0)#usecols= ['_id','text','cleaned_text']
    return data



# + id="KrtDaWEB-ERq" colab_type="code" colab={}
columns=['_id','cleaned_text']
covid_dataset=read_covid_dataset(columns,10)

covid_dataset.to_csv(r'/content/drive/My Drive/COVID-19/TopicModel/covid_data_full_v5_cleaned_dropnan_10.csv',index=False,sep=',',header=True)

# + id="-zKgDJxNz0ql" colab_type="code" outputId="036ef17c-1720-4e95-e199-83d2b5b896dc" colab={"base_uri": "https://localhost:8080/", "height": 390}
covid_dataset


# + id="ArQ5w_WUFZD-" colab_type="code" colab={}
def all_words(covid_dataset):
    all_words=[]
    for line in covid_dataset['cleaned_text']:
        wod_line=line.split(' ')
        for word in wod_line:
            all_words.append(word)
    print(len(all_words))
    return all_words




# + id="_GFNsE1PKzrL" colab_type="code" outputId="2b14a07f-985e-42f2-ed62-2f1f65a99a5e" colab={"base_uri": "https://localhost:8080/", "height": 35}
total_words=all_words(covid_dataset)

# + id="DccYY6nfLOLj" colab_type="code" outputId="a6521f68-b113-4cf5-9ccd-0422460e1756" colab={"base_uri": "https://localhost:8080/", "height": 35}
type(total_words)


# + id="yyGheK4nFJ2C" colab_type="code" outputId="1ab97753-b5d7-4737-9079-c7dd07019fd9" colab={"base_uri": "https://localhost:8080/", "height": 35}
def unique_words(x):
    pure_list=list(dict.fromkeys(x))
    import csv
    with open('/content/drive/My Drive/COVID-19/TopicModel/all_terms_10.csv',mode='w', newline='') as myfile:
        wr = csv.writer(myfile, quoting=csv.QUOTE_NONE,delimiter=',')
        wr.writerow(pure_list)
    return len(pure_list)

mylist = unique_words(total_words)
mylist

# + id="m-KjYqIyPbtf" colab_type="code" outputId="27f052dd-dc99-454e-99aa-e180ed2e64ed" colab={"base_uri": "https://localhost:8080/", "height": 115}
data = pd.read_csv("/content/drive/My Drive/COVID-19/TopicModel/all_terms_10.csv")
data

# + id="2mii2izIV_MW" colab_type="code" outputId="b47a4d0d-2cb3-43f5-9f90-5fe253a6cab0" colab={"base_uri": "https://localhost:8080/", "height": 35}
#data[]
s=data[data.columns[1]]
type(s)
s.name

# + id="9Tntc2fLVxbo" colab_type="code" outputId="f213864a-2f9d-4a35-dc3d-b935a54a3f13" colab={"base_uri": "https://localhost:8080/", "height": 151}
data.where(df =='a')

# + [markdown] id="spS-TQl-9uZ2" colab_type="text"
# ## TF-IDF

# + id="8OSNU-4YBh-D" colab_type="code" outputId="371f221c-a96c-40a5-9d77-0caa83f2e1b3" colab={"base_uri": "https://localhost:8080/", "height": 53}
import nltk
nltk.download('punkt')
def tokenizer(text):
    tokens = nltk.word_tokenize(text)
    return tokens


# + id="jsyEp7sh_lr2" colab_type="code" colab={}
def tfidf():
    from sklearn.feature_extraction.text import TfidfVectorizer
    v = TfidfVectorizer(tokenizer=tokenizer,analyzer ='word')
    x = v.fit_transform(covid_dataset.cleaned_text.values)
    data=x.toarray()
    headers=v.get_feature_names() 
    df = pd.DataFrame (data, columns =headers,index=covid_dataset.index.values)
    df.to_csv(r'/content/drive/My Drive/COVID-19/TopicModel/covid_TFIDF_10.csv')#,index=True,sep=',',header=True)
    return df


# + id="UeypoCdJFWLp" colab_type="code" outputId="6a9ad7dd-4e90-493a-ce45-6d100a074fb0" colab={"base_uri": "https://localhost:8080/", "height": 197}
covid_dataset.index.values

# + id="AX9b0Dnwt6W0" colab_type="code" outputId="ca00d214-69b9-4816-efe3-06b42879f0d6" colab={"base_uri": "https://localhost:8080/", "height": 408}
tfidf()
   

# + id="eJ4R5D6qVSO_" colab_type="code" outputId="0b6f9a4c-2aa7-4391-953f-827ac1524c00" colab={"base_uri": "https://localhost:8080/", "height": 215}
df['corona']

# + [markdown] id="o__uHgxauqEJ" colab_type="text"
# #**CorelationCoeffitien matrix (H)**
# https://towardsdatascience.com/word-embedding-using-bert-in-python-dd5a86c00342

# + [markdown] id="n8HCgjGiD5tF" colab_type="text"
# ##FastText
# https://fasttext.cc/docs/en/english-vectors.html
#
# https://medium.com/@h_bushroh/text-similarity-with-fasttext-word-embeddings-c765d97df682
# https://github.com/facebookresearch/fastText/blob/master/docs/pretrained-vectors.md
#
# https://github.com/facebookresearch/fastText/blob/master/docs/pretrained-vectors.md
#
# https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.en.zip
#
# https://github.com/ncbi-nlp/BioSentVec
#

# + id="3tS6tLMzKfyP" colab_type="code" colab={}
import os
os.chdir('/content/drive/My Drive/COVID-19/TopicModel/')

# + id="-11tfB6IKboO" colab_type="code" outputId="3cd6bd23-bd9f-456e-e9db-5b66ecbbc769" colab={"base_uri": "https://localhost:8080/", "height": 269}
# !ls

# + [markdown] id="lWxli8joOpYT" colab_type="text"
# ###Download

# + id="KTs2jLUxFbuL" colab_type="code" outputId="0d064f1c-d89c-4a7e-cfe6-ed15d9fa2fa0" colab={"base_uri": "https://localhost:8080/", "height": 215}
# !wget https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.en.zip

# + id="pN8OXacQN047" colab_type="code" outputId="54dff748-f2c6-4d01-dc3e-f720eb118e0f" colab={"base_uri": "https://localhost:8080/", "height": 71}
# !unzip '/content/drive/My Drive/COVID-19/TopicModel/wiki.en.zip'

# + [markdown] id="Vw16jYskyrZA" colab_type="text"
# ###Load Model

# + id="EZ6WQ5A2RWnp" colab_type="code" outputId="c367cb91-e3f2-4dd5-f6eb-2442c9b243ad" colab={"base_uri": "https://localhost:8080/", "height": 73}
#https://radimrehurek.com/gensim/models/deprecated/keyedvectors.html
#https://github.com/RaRe-Technologies/gensim/blob/develop/docs/notebooks/Word2Vec_FastText_Comparison.ipynb
#from gensim.models.fasttext import FastText
from gensim.models import Word2Vec, KeyedVectors
fast_text_vec = KeyedVectors.load_word2vec_format('/content/drive/My Drive/COVID-19/TopicModel/wiki.en.vec')
#fast_text_model=FastText.load_fasttext_format('/content/drive/My Drive/COVID-19/TopicModel/wiki.en.vec')

# + id="EJxpDbEv8ex6" colab_type="code" outputId="69028c79-92c8-43b8-92ca-cefa10d9faaf" colab={"base_uri": "https://localhost:8080/", "height": 253}
fast_text_vec.most_similar('zot')

# + id="ojs5sNFCAjdE" colab_type="code" outputId="162e4bc8-0408-4df1-f744-3f5c4e3bffb3" colab={"base_uri": "https://localhost:8080/", "height": 35}
fast_text_vec.cosine_similarities(fast_text_vec.get_vector('dog'),[fast_text_vec.get_vector('cat'),fast_text_vec.get_vector('bit')])

# + id="mMew-MUXEz4q" colab_type="code" colab={}
data = pd.read_csv("/content/drive/My Drive/COVID-19/TopicModel/all_terms_10.csv")
data

# + id="FbqafQTRE-cV" colab_type="code" outputId="aed231c3-f8a9-4658-daa5-9d2b0b5bda42" colab={"base_uri": "https://localhost:8080/", "height": 125}
all_words=data.columns[1:]
len(all_words)
all_words

# + id="o9NZcxjHFtzZ" colab_type="code" outputId="4c25c210-0db1-4ebb-9b85-b67f89f83837" colab={"base_uri": "https://localhost:8080/", "height": 1000}
all_fast_text_vec_list=[]
for w in all_words:
    print(type(w))
    all_fast_text_vec_list.append(fast_text_vec.get_vector(w))

#all_fast_text_vec_list

# + id="sy05cX9QFRfd" colab_type="code" colab={}
for w in all_words:
    

# + id="ZKqNBrG5Ez0q" colab_type="code" colab={}
s=data[data.columns[1]]
type(s)
s.name

# + [markdown] id="FpmlEbRpMdVX" colab_type="text"
# ##Bio word2vec
# https://github.com/ncbi-nlp/BioSentVec
# https://github.com/jakerochlinmarcus/biomedical-word-embeddings

# + [markdown] id="PDPW9ir_Pz3y" colab_type="text"
# ###Download

# + id="lHTrN80oP7zi" colab_type="code" colab={}
import os
os.chdir('/content/drive/My Drive/COVID-19/TopicModel/')

# + id="l4yhPPmJQAWt" colab_type="code" outputId="e1dfb6d7-30b7-4f72-c34d-8475b38615eb" colab={"base_uri": "https://localhost:8080/", "height": 269}
# !ls

# + id="oa8ViHCZEzvi" colab_type="code" outputId="c7918006-6f93-4396-d1d4-336f8ba8bb00" colab={"base_uri": "https://localhost:8080/", "height": 215}
# !wget https://ftp.ncbi.nlm.nih.gov/pub/lu/Suppl/BioSentVec/BioWordVec_PubMed_MIMICIII_d200.vec.bin

# + id="XfwfteT3Yj1S" colab_type="code" outputId="65d14e6f-daf0-4232-fdaa-26e0d779e8ab" colab={"base_uri": "https://localhost:8080/", "height": 215}
# !wget http://evexdb.org/pmresources/vec-space-models/PubMed-and-PMC-w2v.bin

# + [markdown] id="3FeXVpUSR531" colab_type="text"
# ###Load

# + id="ImEgtjq5lIsa" colab_type="code" outputId="1e06fca4-2386-4999-b6c7-a05ca8f10d01" colab={"base_uri": "https://localhost:8080/", "height": 35}
# !ls

# + id="KwCza2MGlONB" colab_type="code" colab={}
# !mv ./PubMed-and-PMC-w2v.bin '/content/drive/My Drive/COVID-19/TopicModel/'

# + id="6z9ghyHfR7mM" colab_type="code" outputId="8f07abca-9166-4217-f87e-387b0603c01a" colab={"base_uri": "https://localhost:8080/", "height": 73}
from gensim.models import Word2Vec, KeyedVectors
Bio_text_vec = KeyedVectors.load_word2vec_format('/content/drive/My Drive/COVID-19/TopicModel/PubMed-and-PMC-w2v.bin', binary=True)

# + id="tZmFy5f4FhuF" colab_type="code" outputId="e8f1665f-4d6f-4f06-d739-9de3dca4939c" colab={"base_uri": "https://localhost:8080/", "height": 253}
Bio_text_vec.most_similar('zot')


# + id="9b8sSEuGJgbE" colab_type="code" outputId="89ec5557-b2ed-4161-8661-6944760367eb" colab={"base_uri": "https://localhost:8080/", "height": 115}
def H1():
    data = pd.read_csv("/content/drive/My Drive/COVID-19/TopicModel/all_terms_10.csv")
    all_words=data.columns[1:]
    len(all_words)
    all_words
    all_Bio_text_vec_list=[]
    Bio_text_word=[]
    Not_Bio_text_word=[]
    for w in all_words:
        #print(type(w))
        try:
            all_Bio_text_vec_list.append(Bio_text_vec.get_vector(w))
            Bio_text_word.append(w)
        except:
            Not_Bio_text_word.append(w)
            pass
    print(len(all_Bio_text_vec_list))#5751
    print(len(Bio_text_word))#5751
    print(len(Not_Bio_text_vec_list))#508
    all_Bio_word_cos=[]
    for w in Bio_text_word:
        cos=Bio_text_vec.cosine_similarities(Bio_text_vec.get_vector(w),all_Bio_text_vec_list)
        all_Bio_word_cos.append(cos)
    df_H1 = pd.DataFrame(all_Bio_word_cos, index =Bio_text_word,columns=Bio_text_word)
    df_H1.to_csv(r'/content/drive/My Drive/COVID-19/TopicModel/H1_10doc_Bio.csv')
    return df_H1



# + id="cbWox1VUql0c" colab_type="code" outputId="ac413ebf-c2ea-48a1-f2e6-90d46726dbe4" colab={"base_uri": "https://localhost:8080/", "height": 439}
H1()


# + id="lDgoe5RW3aN3" colab_type="code" colab={}
def H2_50():
    data_H1 = pd.read_csv('/content/drive/My Drive/COVID-19/TopicModel/H1_10doc_Bio.csv',index_col=0)#,header=None)
    for col in data_1.columns:
        data_H1.loc[data_H1[col] < .50, col] = 0
    data_H1.to_csv(r'/content/drive/My Drive/COVID-19/TopicModel/H2_10doc_Bio_50.csv')
    return data_H1


# + id="RE3jXE4A3yge" colab_type="code" outputId="cc7b709b-53f3-4dbd-d025-12c17deda3a4" colab={"base_uri": "https://localhost:8080/", "height": 439}
H2_50()

# + id="338Q5MA-5FtD" colab_type="code" outputId="e61b5858-4269-46a7-9083-026f22fe0f5c" colab={"base_uri": "https://localhost:8080/", "height": 233}
#(data_1 == 0).astype(int).sum(axis=1)

# + [markdown] id="958l3WS27rNW" colab_type="text"
# #**Topic Doument**

# + [markdown] id="C6TPJyXm_lCe" colab_type="text"
# ##Inverse Martix H

# + id="qPqft5H98WMV" colab_type="code" colab={}
v=wHT--->wv/H or w=vHi


# + id="1A_Fu4LL8cFa" colab_type="code" colab={}
def inv_H2():
    data_H2 = pd.read_csv('/content/drive/My Drive/COVID-19/TopicModel/H2_10doc_Bio_50.csv',index_col=0)
    df_inv = pd.DataFrame(np.linalg.pinv(data_H2.values), data_H2.columns, data_H2.index)
    df_inv.to_csv(r'/content/drive/My Drive/COVID-19/TopicModel/H2_Inv_Bio_50.csv')
    return df_inv


# + id="dDopbZKF8cbW" colab_type="code" outputId="7e3bb1d4-3f77-4164-a3b2-d20b2d331860" colab={"base_uri": "https://localhost:8080/", "height": 609}
inv_H2()


# + id="VT8Y9PIy9Bsy" colab_type="code" colab={}
def  W():
    inv_H2 = pd.read_csv('/content/drive/My Drive/COVID-19/TopicModel/H2_Inv_Bio_50.csv',index_col=0)
    inv_H2.sort_index(axis=1, inplace=True)
    inv_H2.sort_index(axis=0, inplace=True)
    tfIdf_bio = pd.read_csv('/content/drive/My Drive/COVID-19/TopicModel/covid_TFIDF_10.csv',index_col=0)
    tfIdf_bio_2=tfIdf_bio[inv_H2.columns]
    W=tfIdf_bio_2.dot(inv_H2)
    W.index=tfIdf_bio.index
    W.to_csv(r'/content/drive/My Drive/COVID-19/TopicModel/W_Bio_50.csv')
    return W


# + id="hJpjm3xSCl8u" colab_type="code" outputId="cb8bf18b-636a-41f4-f161-c39b09c5671d" colab={"base_uri": "https://localhost:8080/", "height": 55}
'''inv_H2 = pd.read_csv('/content/drive/My Drive/COVID-19/TopicModel/H2_Inv_Bio_50.csv',index_col=0)
inv_H2.sort_index(axis=1, inplace=True)
inv_H2.sort_index(axis=0, inplace=True)
tfIdf_bio = pd.read_csv('/content/drive/My Drive/COVID-19/TopicModel/covid_TFIDF_10.csv',index_col=0)#,usecols=inv_H2.columns)
tfIdf_bio_2=tfIdf_bio[inv_H2.columns]
tfIdf_bio_2'''

# + id="plnX0ptwYqet" colab_type="code" colab={}
#tfIdf_bio_2.dot(inv_H2)

# + id="a4o-lhcrTHIP" colab_type="code" outputId="d58c33ee-ac90-49cb-fc12-c36cc58c88e2" colab={"base_uri": "https://localhost:8080/", "height": 578}
#tfIdf_bio[inv_H2.columns]
W()# Doc-topic() topic is column, Doc is row)

# + id="fkIO5869C4ET" colab_type="code" colab={}


# + [markdown] id="KAsoaDaSPiPB" colab_type="text"
# #**Searching**

# + id="JDjVm3w5RXfA" colab_type="code" outputId="88860f7c-3bac-4bee-953b-1757aa79f478" colab={"base_uri": "https://localhost:8080/", "height": 73}
from gensim.models import Word2Vec, KeyedVectors
Bio_text_vec = KeyedVectors.load_word2vec_format('/content/drive/My Drive/COVID-19/TopicModel/PubMed-and-PMC-w2v.bin', binary=True)

# + id="cCqHGOLNoy5F" colab_type="code" colab={}
search="coronavirus,corona virus,covid-19,2019-ncov,ncov,sars-cov-2,sars,sars-cov,mers,mers-cov,severe acute respiratory syndrome,middle east respiratory syndrome"
all_search_list=search.split(",")
Search_NOT="animal,equine,porcine,calves,dog,canine,feline,bat,camel"
all_Search_NOT_list=Search_NOT.split(",")
Search_AND="incubation period,onset,exposure"
all_Search_AND_list=Search_AND.split(",")


# + id="2SZsirKBRI8N" colab_type="code" colab={}


# + id="g8hVnsX_Rfc6" colab_type="code" outputId="6c40d72e-c06f-4bb5-9e29-ec91633d724e" colab={"base_uri": "https://localhost:8080/", "height": 91}

all_most_similar=dict()
for w in all_search_list:
    try:
        s=Bio_text_vec.most_similar(w.lower())
        #search_list.append(w)
        #print(w)
        #print(s)
        most_similar_list=[]
        for i in range(len(s)):
            if s[i][1]>=.5:
                #print(s[i][1])
                most_similar_list.append(s[i])
        all_most_similar[w]=most_similar_list

    except:
        #print("Not")
        pass
print("Research topic",all_most_similar)


# + id="797ASEraRIrq" colab_type="code" colab={}
doc_topic_df = pd.read_csv('/content/drive/My Drive/COVID-19/TopicModel/W_Bio_50.csv',index_col=0)

# + id="3eH3ShwJa1pA" colab_type="code" outputId="ff7d1546-9b16-46bd-d322-9944cc50e746" colab={"base_uri": "https://localhost:8080/", "height": 35}
print(all_most_similar.get(1))

# + id="lrbgCqbAa8vP" colab_type="code" colab={}


StarcoderdataJupyterScriptsDedupFiltered