# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="0qW5ai19Slq_" colab_type="text"
# # **Import and other Reqs**

# + id="qpugzmJTKlam" colab_type="code" outputId="26af5e76-db86-48bb-891f-63f1f3150f6f" colab={"base_uri": "https://localhost:8080/", "height": 268}
# !pip install wikipedia

# + id="b2HO8XOAIBfJ" colab_type="code" colab={}
import spacy
from spacy import displacy
import wikipedia
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
# %matplotlib inline

# + id="MB9bMM1OIUhX" colab_type="code" colab={}
nlp = spacy.load('en') #en for english, fr for french and so on

# + id="1CBsTwMOzLG-" colab_type="code" colab={}
from spacy.lang.en.stop_words import STOP_WORDS

# + id="TDfwhHUNopBe" colab_type="code" colab={}
import warnings
warnings.filterwarnings("ignore")

# + id="o9p_f7ZVJPQt" colab_type="code" outputId="00fe1f44-0271-41df-ae04-b3ae3b614839" colab={"base_uri": "https://localhost:8080/", "height": 54}
from google.colab import drive
drive.mount('/gdrive')
# %cd /gdrive

# + id="Y-REEROEeG3b" colab_type="code" colab={}
"""
!ls   #similar to linux ls command
%cd ..   #similar to linux directory navigation
%cd 'My Drive/CIS_508/Colab Notebooks/NER' 
"""

# + [markdown] id="ak6P7jDQJmK2" colab_type="text"
# # **Read text document**

# + id="bKi8A9mPIbjI" colab_type="code" outputId="a65464ca-7701-4eae-83e3-10c74de31b8b" colab={"base_uri": "https://localhost:8080/"}
doc = nlp("SpaCY is a cool tool")
doc

# + [markdown] id="ezcrnqCxJq7f" colab_type="text"
# # **Read from a file**

# + id="pLQBfUsGIh86" colab_type="code" outputId="fe0885f5-3662-4e9e-e5ff-07ce3547b34e" colab={"base_uri": "https://localhost:8080/", "height": 90}
myfile = open('/gdrive/My Drive/CIS_508/Colab Notebooks/NER/trial.txt').read()
docfile = nlp(myfile)
docfile

# + [markdown] id="_6bVfj-5KfEr" colab_type="text"
# # **Read from wikipedia**

# + id="Ye_GInrDKeuS" colab_type="code" colab={}
page = wikipedia.page("Vinodhini")
page_url = page.url
page_title = page.title
content = page.content

wikitext = nlp(content)

# + [markdown] id="q0_GPDnGNs10" colab_type="text"
# # **Explore the basic functions**

# + id="2NY56KxiNtAj" colab_type="code" outputId="479b9768-e844-4cb0-c176-ef6a6023f2f0" colab={"base_uri": "https://localhost:8080/"}
print(docx)
print("\n",docfile)

# + id="3tqJpMiyN0z-" colab_type="code" outputId="cd046fea-09e8-499c-c5de-8a276bfd5547" colab={"base_uri": "https://localhost:8080/"}
print(docx.text)
print(docfile.text)

# + [markdown] id="b-V4GJjdJ1Nw" colab_type="text"
# # **Tokenization - Sentence Tokens**

# + id="OpDuCDY7J0vX" colab_type="code" outputId="bf237dd5-5c58-428d-d2fc-d8976ac99feb" colab={"base_uri": "https://localhost:8080/"}
num = 1
for sentence in docfile.sents:
  print(str(num), ": ", sentence)
  num += 1

# + id="FOMQqEdQJjwI" colab_type="code" outputId="c97b67a9-2521-431d-f25f-9c5e0d80137e" colab={"base_uri": "https://localhost:8080/"}
for num, sentence in enumerate(docfile.sents):  #enum starts from 0
  print(f'{num}: {sentence}')

# + [markdown] id="fQyO5YjkL9pS" colab_type="text"
# # **Tokenization - Word Tokens**

# + id="fCBw9BvwL4fU" colab_type="code" outputId="cbe5fb06-d1ca-4735-c1e5-3b91a27902fa" colab={"base_uri": "https://localhost:8080/"}
for word in docx:
  print(word.text) #word.text and word are the same here 

#get a list of word tokens
words = [word for word in docx]
words

# + id="bPVHgjxZMIll" colab_type="code" colab={}
for word in docfile:
  print(word)

words = [word.text for word in docfile]
words

# + id="p-6C66-LMIrl" colab_type="code" colab={}
#split using spaces are not exactly same because word tokenizer takes care of the punctuation and other nuances. But the basic idea is that 
docfile.text.split(" ")

# + [markdown] id="7mYmlijvSdZo" colab_type="text"
# # **Other word functions**

# + id="WZlMj4GlMIwx" colab_type="code" colab={}
for word in docfile:
  print(word.text, word.shape_)

# + id="W-IUxKXoMIuZ" colab_type="code" outputId="6769f8b2-b12c-4a51-d459-414cd2b23cc4" colab={"base_uri": "https://localhost:8080/"}
for word in docx:
  print(word.text, word.shape_, word.is_alpha, word.is_stop) #word.shape is not human readable format

# + [markdown] id="4o6S2N_eU382" colab_type="text"
# # **POS Tagging**

# + id="O_rCnmt5MIzO" colab_type="code" outputId="70d71ffc-c8e5-4961-b5df-8155e5ba591d" colab={"base_uri": "https://localhost:8080/"}
for word in docx:
  print(word.text, word.pos_) #word.pos is not in human readable form

# + id="ER734GPZVgRb" colab_type="code" outputId="f9aa0e9c-d577-4790-a0cd-4db951f1dcd3" colab={"base_uri": "https://localhost:8080/"}
print(spacy.explain('DET'))
print(spacy.explain('PROPN'))

# + id="p6ALETsWVgUF" colab_type="code" outputId="511af52e-ba10-4c61-8552-ea8c2cbc50ad" colab={"base_uri": "https://localhost:8080/"}
#Now let us mess with Spacy and see if it is able to tag the infamous sentence
buff = nlp('Buffalo buffalo Buffalo buffalo buffalo buffalo Buffalo buffalo')
buff

# + [markdown] id="yXw_IyhVWhxM" colab_type="text"
# ![image.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAVgAAAFBCAYAAAAytbcTAAAgAElEQVR4Ae19C8weRfU+EKWIl5iGkqZUo02MyQ8VJBpNFC8xaPL7g9oQElTyNSEkhp9oovEGUloEK3douRUo5VakBUsplDtFaEFoKVbukCLQG/dSEITvK8L888zu2T07776X3W8vs/s+k3zf7rszO3PmmTPPnp2dObOTYSACRIAIEIFSENiplFyZKREgAkSACBgSLJWACBABIlASAiTYkoBltkSACBABEix1gAgQASJQEgKpBHv/uocM/4gBdYA6QB3orgODcHJXgh0d22H4RwyoA9QB6kCnDuDBM0ggwfJBwgcpdYA6kFEHSLAZAeNTuvMpTUyICXUgXQcKJViOw3QfhyE2xIY6MBw6oB82hRPsIOMNTEMEiAARaCMCIFQSbBtblnUiAkSgdgRIsLU3AQUgAkSgrQiQYNvasqwXESACtSNAgq29CSgAESACbUWABNvWlmW9iA<KEY>)

# + id="TchFxLwLVgWm" colab_type="code" outputId="77b33f20-50fa-4e2f-cc8d-6a7c4342adf9" colab={"base_uri": "https://localhost:8080/"}
for word in buff:
  print(word.text, word.pos_)

# Ofcourse it doesn't tag it right, because it is a very complicated sentence 

# + id="Vdry4zEuVgaA" colab_type="code" outputId="f6ed55f0-0346-43df-bbd6-f47e85d390b5" colab={"base_uri": "https://localhost:8080/"}
for word in nlp('I fish a fish'):
  print(word.text, word.pos_, word.tag_)   

#spacy got this sentence right

# + id="5NLTWtN8MI3h" colab_type="code" outputId="6bad461e-8c5e-451c-a084-9774f63f845c" colab={"base_uri": "https://localhost:8080/"}
spacy.explain('VBP')

# + id="hY9NgAduXScx" colab_type="code" outputId="01c55ea5-d925-4a59-e9fe-ede91f183a1b" colab={"base_uri": "https://localhost:8080/"}
for word in nlp('All the faith I had had no effect on the outcome of my life'):
  print((word.text, word.pos_, word.tag_)) 

# + id="DpDnFoi4XSfR" colab_type="code" outputId="e440e58d-774e-4f59-e373-a76c6b4d684e" colab={"base_uri": "https://localhost:8080/"}
print(spacy.explain('VBD'))
print(spacy.explain('VBN'))

# + [markdown] id="-xzDiRX2Zs7Y" colab_type="text"
# # **Syntactical Dependency**

# + id="cBbbGAViXSiM" colab_type="code" outputId="8e1d2f49-c84c-40e5-db12-2254bd3cda5a" colab={"base_uri": "https://localhost:8080/"}
for word in nlp('<NAME> Rose'):
  print(word.text, word.pos_, word.tag_, word.dep_)

# + id="bx1bBK0oXSkP" colab_type="code" outputId="57eb2564-3ec4-495b-98e9-60ea449d1171" colab={"base_uri": "https://localhost:8080/"}
spacy.explain('nsubj')

# + id="RZQDMuFfXSmy" colab_type="code" outputId="a79b76a6-9ca7-43d3-da08-4dbfae1c35d9" colab={"base_uri": "https://localhost:8080/"}
displacy.render(nlp('Jack likes Rose'), style = 'dep', jupyter = True)

# + [markdown] id="e1JPY204aG20" colab_type="text"
# # **Lemmatization**
#
# Stemming just cuts the suffix and prefix wheras lemmatization turns words into meaningful root words

# + id="zE_HRonTaKHp" colab_type="code" outputId="8a595a37-e819-4cdd-a71a-a3f402c13ff4" colab={"base_uri": "https://localhost:8080/"}
doc = nlp("Study studying studious studio student sturdy")
for word in doc:
  print(word.text, word.lemma_, word.pos_) 

# + id="zGTblzTkaKMq" colab_type="code" outputId="91f3de35-4f63-474a-96f2-a1ab3022ccce" colab={"base_uri": "https://localhost:8080/"}
doc = nlp("She walks regularly. walking walks walked walker walkman was were be is")
for word in doc:
  print(word.text, word.lemma_, word.pos_) 

#she walks (walks) is identified as verb wheras the second walks is identified as noun. This is really good

# + id="fBL7fjf8aKPg" colab_type="code" colab={}


# + [markdown] id="Rx1ovjHvcD4I" colab_type="text"
# # **Named Entity Recognition**

# + id="3jhF3suqaKR6" colab_type="code" outputId="1b068700-f733-415e-f557-8b4bdb3ef898" colab={"base_uri": "https://localhost:8080/"}
for word in docfile.ents:
  print(word.text, word.label_)

print("\n",docfile)

# + id="ttPZ1QaBaKUg" colab_type="code" outputId="c5eab96d-a357-4220-e902-ccd6d833c328" colab={"base_uri": "https://localhost:8080/"}
d = nlp("Facebook, America, Explosion.ai")
print(d)
for word in d.ents:
  print(word.text, word.label_)

# + id="VKLHvAaWaKYn" colab_type="code" outputId="c7b94eed-949a-47af-8d0c-67c31a9ce418" colab={"base_uri": "https://localhost:8080/"}
displacy.render(docfile, style = 'ent', jupyter = True)

# + [markdown] id="vYB2O6Z3eA-5" colab_type="text"
# # **Visualizing with Displacy**

# + id="7QkHI2EKeGt7" colab_type="code" outputId="cfbf5ff7-43f9-445d-b4af-ec7a982931fb" colab={"base_uri": "https://localhost:8080/"}
displacy.render(docfile, style = 'ent', options = options, jupyter = True)  #doesn't change for text display

# + id="NV8PSWLceGwm" colab_type="code" colab={}
options = {'compact':True, 'bg':'cornflowerblue', 'color':'#fff', 'font': 'Verdana'}

# + id="lrH6Nys9aKW-" colab_type="code" outputId="d3161cd9-ad82-41ba-954f-a5b6746c4867" colab={"base_uri": "https://localhost:8080/"}
displacy.render(docfile, style = 'dep', options = options, jupyter = True)   #works well for pictorial display

# + id="MLDQDhoFeGy4" colab_type="code" colab={}
svg = displacy.render(docfile, style = 'dep', options = options, jupyter = False)
with open('/gdrive/My Drive/CIS_508/Colab Notebooks/NER/Dependency.svg', 'w', encoding = 'utf-8') as f:
  f.write(svg)

# + [markdown] id="SB40Nx_riS0o" colab_type="text"
# # **Semantic Similarity**
#
# Use case: Recommendation systems

# + id="iTt9QaHweG5Z" colab_type="code" colab={}
dog = nlp("dog")
wolf = nlp("wolf")
elephant = nlp("elephant")
cat = nlp("cat")

# + id="bo7SjozIiSLY" colab_type="code" outputId="122ab031-6ccd-467e-a254-b43cc33204e0" colab={"base_uri": "https://localhost:8080/"}
dog.similarity(cat)  #higher the value more the similarity

# + id="sMaaSH3YiSOM" colab_type="code" outputId="561f8a4a-fdd6-4845-c706-6f04044c16c3" colab={"base_uri": "https://localhost:8080/"}
dog.similarity(wolf)

# + id="IEkpoLy6iSVk" colab_type="code" outputId="0c3f436b-68f2-4edd-9fa7-a59ad8fa91ac" colab={"base_uri": "https://localhost:8080/"}
dog.similarity(elephant)

# + id="YnrDj2mRiSRA" colab_type="code" colab={}
smart = nlp("smart")
clever = nlp("clever")
foolish = nlp("foolish")

# + id="6u4vIbjmeG2a" colab_type="code" outputId="a2cf3853-2552-4e64-a606-cee6b3a30d09" colab={"base_uri": "https://localhost:8080/"}
smart.similarity(clever)

# + id="zRoqb5ExaKKN" colab_type="code" outputId="4b2bee23-fe92-49c6-e3de-7b0c8b198fbf" colab={"base_uri": "https://localhost:8080/"}
clever.similarity(foolish)  #weirdly clever and foolish are more similar than smart and clever.

# + id="ug7K1mClnj_h" colab_type="code" outputId="dfe42550-0095-4da5-b92f-7f194fdda4fc" colab={"base_uri": "https://localhost:8080/"}
data = nlp("mom dad cat dog")
for i in data:
  for j in data:
    print((i, j, i.similarity(j)))

# + id="AgVh0hQ6vKfw" colab_type="code" outputId="31956c61-0475-4653-9827-cc57b6546b1a" colab={"base_uri": "https://localhost:8080/"}
data_sim = [(i.text, j.text, i.similarity(j)) for j in data for i in data]
type(data_sim[0][0])

# + id="gMFqKRDrnkCN" colab_type="code" outputId="ad8f8f9f-1aaf-4ee9-bc51-46634a374db9" colab={"base_uri": "https://localhost:8080/"}
df = pd.DataFrame(data_sim, columns = ['Token 1', 'Token 2', 'Similarity'])
type(df['Token 1'][0])

# + id="dqeJ8xXXvm3h" colab_type="code" outputId="c68369e8-6939-4318-a6c8-2930218ef66b" colab={"base_uri": "https://localhost:8080/"}
df_viz = df.replace({'mom':0, 'dad':1, 'cat':2, 'dog':3})
df_viz

# + id="uO0erp1hnkEm" colab_type="code" outputId="24344aaa-b0c0-4765-f218-26a1aa1b9785" colab={"base_uri": "https://localhost:8080/"}
fig, ax = plt.subplots()
sns.heatmap(df_viz.corr(), annot = True)
plt.show()
#this makes no sense to me

# + id="kd60GHf8xwrf" colab_type="code" outputId="7d497abd-0f23-4a17-a0aa-2dae72fa980d" colab={"base_uri": "https://localhost:8080/"}
fig, ax = plt.subplots()
sns.heatmap(df_viz, annot = True)
plt.show()

# + [markdown] id="WqAm5_ddx-Bv" colab_type="text"
# # **StopWords Processing**

# + id="1i6tznZ2x9Bj" colab_type="code" outputId="4e822b0a-9fa3-4f4d-bdf7-6c1909906f95" colab={"base_uri": "https://localhost:8080/"}
print(STOP_WORDS)

# + id="c0-kp7OVx9Gt" colab_type="code" outputId="4c189ca1-61fb-47f5-929e-a40b47ca9a8c" colab={"base_uri": "https://localhost:8080/"}
for word in docx:
  print(word.text, word.is_stop)

# + id="VDFnOtcfx9JG" colab_type="code" outputId="00ee34f9-138c-4f47-862e-7cfacb6a17a4" colab={"base_uri": "https://localhost:8080/"}
nlp.vocab["Hamburger"].is_stop

# + id="tFHy2d0Kx9NO" colab_type="code" outputId="c152174f-54e2-489b-c39e-cae323ce54b1" colab={"base_uri": "https://localhost:8080/"}
for word in docx:
  if word.is_stop:
    print(word.text)

# + id="NW-8bNStx9R0" colab_type="code" outputId="ac26e396-1322-41dc-e96c-8b71260092f1" colab={"base_uri": "https://localhost:8080/"}
non_stop_words = [ word for word in docx if word.is_stop == False]
non_stop_words

# + id="zVl-KWy1x9Ya" colab_type="code" outputId="bcc89f94-4d56-41f6-e7eb-9d5868f87d69" colab={"base_uri": "https://localhost:8080/"}
lol = nlp("lol")
for word in lol:
  print(word.is_stop)

# + [markdown] id="05usXAgB09Xd" colab_type="text"
# Add new stop words to the list

# + id="8k5I3eJ43eMp" colab_type="code" outputId="40a3e1ce-f331-4755-ff1b-1ac7cda20521" colab={"base_uri": "https://localhost:8080/"}
#In normal spacy vocabulary for enlish language London and cousins are not stop_words
for word in docfile:
  if word.is_stop:
    print((word.text, word.is_stop))

# + id="gtAO8oDbx9Uu" colab_type="code" colab={}
custom_stop_words = ["London", "cousins"] 

# + id="Ra06SB-o3s0W" colab_type="code" colab={}
#we have manually over-written the is_stop for those custom stop words
for word in custom_stop_words:
  nlp.vocab[word].is_stop = True

# + id="9vtvByYWx9L5" colab_type="code" outputId="d6501486-b97e-48fe-c422-13ccf3a53a80" colab={"base_uri": "https://localhost:8080/"}
#now London and cousins appear in the stop words list
for word in docfile:
  if word.is_stop:
    print((word.text, word.is_stop))

# + id="9OO6QOD-x9FU" colab_type="code" colab={}
#reset London and cousins back
for word in custom_stop_words:
  nlp.vocab[word].is_stop = False

# + [markdown] id="i-eoStWU5quE" colab_type="text"
# # **Noun Chunks**

# + id="AqA4Wi1047mF" colab_type="code" colab={}
doc = nlp("The man, although he had good looks, kept sulking about his fat body")

# + id="DmH448ab47on" colab_type="code" outputId="a33ec1a6-e16b-410b-caee-447fdc6d210b" colab={"base_uri": "https://localhost:8080/"}
#retrieve chunks of nouns like below
for token in doc.noun_chunks:
  print(token.text)

# + id="NS8f6EV447vy" colab_type="code" outputId="716a8ec8-6025-4ea2-8505-c35271cc575e" colab={"base_uri": "https://localhost:8080/"}
#get to the root word of the chunks
for token in doc.noun_chunks:  
  print(token.root.text)

# + id="LnPDUxj047tq" colab_type="code" outputId="044a8c78-8397-43af-93b0-bbe17c3ed7a9" colab={"base_uri": "https://localhost:8080/"}
#head of the nouns
for token in doc.noun_chunks:
  print(token.root.text, "..", token.root.head.text)


# + [markdown] id="bLuii2Ds7s1f" colab_type="text"
# # **Sentence Boundary detection**

# + id="rbcta0NU47rK" colab_type="code" colab={}
def mycustom_boundary(docx):
  for token in docx[:-1]:
    if (token.text == '---'):
       docx[token.i+1].is_sent_start = True
  return docx


# + id="6pElsRpu7ySV" colab_type="code" colab={}
nlp.add_pipe(mycustom_boundary, before='parser')

# + id="wr17TmoY7yUx" colab_type="code" colab={}
#figure out what is wrong
my_sent = nlp(u"Helloooo --- what are you --- doing")
for s in my_sent.sents:
  print(s)

# + id="TyDQi7Ax7yY6" colab_type="code" colab={}


StarcoderdataJupyterScriptsDedupFiltered