# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import os
import json
from docx import Document
from io import StringIO, BytesIO
import re

import pandas as pd
import json
import spacy
from nltk.corpus import stopwords

from gensim.models import LdaModel
from gensim.models.wrappers import LdaMallet
import gensim.corpora as corpora
from gensim.corpora import Dictionary
from gensim import matutils, models
from gensim.models import CoherenceModel
import pyLDAvis.gensim

from docx import Document
from io import StringIO, BytesIO
import matplotlib.pyplot as plt
# %matplotlib inline
plt.rcParams.update({'font.size': 14, 'lines.linewidth': 3})
# -

notebook_dir = os.getcwd()

event_type = 'ANIMAL'
with open(F'../data/sop_jsons/{event_type}.txt') as f:
    drugs = json.load(f)
f.close()
type(drugs)

juris, roles, sops = list(), list(), list()
for juri, dct in drugs.items():
    for role, sop in dct.items():
        juris.append(juri)
        roles.append(role)
        sops.append(sop)
drugsdf = pd.DataFrame({'juri': juris, 'role': roles, 'sop': sops})
drugsdf.head(3)

drugs_calltaker = drugsdf[drugsdf['role'] == 'call taker']
drugs_dispatcher = drugsdf[drugsdf['role'] == 'dispatcher']

nlp = spacy.load("en_core_web_sm")
# stop_words = set(stopwords.words('english'))

remove = [
    "subject:",
    "writes:",
    "wrote:",
    "write:"
]
def multiple_replace(text, unwanted = remove):
    """
    Replace unwanted pattern with a space; return a string
    """ 
    reg = re.compile(f"({'|'.join(remove)})")
    return reg.sub(" ", text)
"(%s)" % "|".join(remove) == f"({'|'.join(remove)})"


def preprocess(strlist,
               min_token_len = 2,
               allowed_pos = ['ADV', 'ADJ', 'VERB', 'NOUN']): 
#     texts = ' '.join(strlist).lower()
    res = list()
    for string in strlist:
        text = string.lower()
        doc = nlp(text)
        res += [token.lemma_ for token in doc \
               if token.is_alpha \
               # Spacy considers 'call' as a stop word, which is not suitable for our case
               and not token.is_stop \
#                and token.text not in stop_words \              
               and token.pos_ in allowed_pos \
               and len(token.lemma_) > min_token_len]
    
    return ' '.join(res)


drugs_calltaker.shape

['mama mele maso'.split(), 'ema ma mama'.split()]


def get_dct_dtmatrix(sops):
    corpus = [sop.split() for sop in map(preprocess, sops)]
    dictionary = corpora.Dictionary(corpus)
    doc_term_matrix = [dictionary.doc2bow(doc) for doc in corpus]
    return doc_term_matrix, corpus, dictionary


# +
# drugs_calltaker['sop']
# -

doc_term_matrix, corpus, dictionary = get_dct_dtmatrix(drugs_calltaker['sop'])

corpus[0]


def topics_with_coherence(doc_term_matrix, corpus, dictionary, texts, 
                          N = 10, passes = 20, coherence = 'c_v', random_state = 2020):
    num_topic, ldas, scores = list(), list(), list()
#     doc_term_matrix = [dictionary.doc2bow(doc) for doc in corpus]
    for n in range(1, N+1):
        lda = models.LdaModel(corpus = doc_term_matrix, 
                              id2word = dictionary, 
                              num_topics = n, 
                              passes = passes, 
                              random_state = random_state)
        coherence_model = CoherenceModel(
            model = lda, 
            texts = corpus, 
            dictionary = dictionary, 
            coherence = coherence)
        coherence_score = coherence_model.get_coherence()
#         mod = {'num_topic': n, 'model': lda, 'coherence_score': coherence_score}
#         res[f"num_topic={n}"] = [lda, coherence_score]
        num_topic.append(n)
        ldas.append(lda)
        scores.append(coherence_score)
    return pd.DataFrame({
        'num_topic':num_topic,
        'model': ldas,
        'coherence_score': scores
    })


drugs_call_coherence_cv = topics_with_coherence(doc_term_matrix, corpus, dictionary, drugs_calltaker['sop'])
drugs_call_coherence_cv

fig, ax = plt.subplots(1, 1, figsize = (12, 8))
ax.plot(drugs_call_coherence_cv.loc[:, 'num_topic'].values, 
        drugs_call_coherence_cv.loc[:, 'coherence_score'].values)
ax.set_xlabel('number of topics')
ax.set_ylabel('coherence score')
ax.set_title('Coherence Score (c_v) vs Number of Topics')
plt.show()

chosen_model_cv = list(drugs_call_coherence_cv.iloc[1:, :].sort_values('coherence_score')['model'])[-1]
print(chosen_model_cv)

chosen_model_cv.print_topics()

model_cv_2 = list(drugs_call_coherence_cv[drugs_call_coherence_cv['num_topic'] == 2]['model'])[0]


def get_topic(model, doc):
    ppdoc = preprocess(doc)
    doc_term_arr = dictionary.doc2bow(ppdoc.split())
    return sorted(model[doc_term_arr], 
                  key = lambda x: x[1], 
                  reverse = True)[0][0]


sent = list(drugs_calltaker['sop'])#.append(['aa', 'bb'])
sent.append(['This is a test', 'to see if the model works'])
# sent

preprocess(list(drugs_calltaker['sop'])[2])

ppsent = preprocess(sent[1])
doc_term_arr = dictionary.doc2bow(ppsent.split())
chosen_model_cv[doc_term_arr]
# doc_term_arr

list(map(lambda x: get_topic(chosen_model_cv, list(x)), sent))

# +
# umass_6 = list(drugs_call_coherence_umass[drugs_call_coherence_umass['num_topic'] == 4]['model'])[0]
# print(umass_6)
# list(map(lambda x: get_topic(umass_6, list(x)), sent))
# -

drugs_call_coherence_umass = topics_with_coherence(doc_term_matrix, corpus, dictionary, 
                                                   drugs_calltaker['sop'], coherence = 'u_mass')
fig, ax = plt.subplots(1, 1, figsize = (12, 8))
ax.plot(drugs_call_coherence_umass.loc[:, 'num_topic'].values, 
        drugs_call_coherence_umass.loc[:, 'coherence_score'].values)
ax.set_xlabel('number of topics')
ax.set_ylabel('coherence score')
ax.set_title('Coherence Score (u_mass) vs Number of Topics')
plt.show()

drugs_call_coherence_cuci = topics_with_coherence(doc_term_matrix, corpus, dictionary, 
                                              drugs_calltaker['sop'], coherence = 'c_uci')
fig, ax = plt.subplots(1, 1, figsize = (12, 8))
ax.plot(drugs_call_coherence_cuci.loc[:, 'num_topic'].values, 
        drugs_call_coherence_cuci.loc[:, 'coherence_score'].values)
ax.set_xlabel('number of topics')
ax.set_ylabel('coherence score')
ax.set_title('Coherence Score (c_uci) vs Number of Topics')
plt.show()

drugs_call_coherence_cnpmi = topics_with_coherence(doc_term_matrix, corpus, dictionary, 
                                        drugs_calltaker['sop'], coherence = 'c_npmi')
fig, ax = plt.subplots(1, 1, figsize = (12, 8))
ax.plot(drugs_call_coherence_cnpmi.loc[:, 'num_topic'].values, 
        drugs_call_coherence_cnpmi.loc[:, 'coherence_score'].values)
ax.set_xlabel('number of topics')
ax.set_ylabel('coherence score')
ax.set_title('Coherence Score (c_npmi) vs Number of Topics')
plt.show()

# #### Reflection of DRUGS coherence score
# - the coherence score is very high for the one-topic model
# - this makes sense, because we are looking at docs under the same type "DRUGS"
#
# #### Question
# - While the model assigns the documents with the correct topic, does this necessarily mean the documents are similar enough to be consolicated?
# - LDA in not stable.  How will this instability affect us?

# +
# os.listdir('.')

# +
# json_dir = '../data/interim'
# os.chdir(json_dir)
# all_sop = os.listdir('.')
# print(all_sop)

# +
# def load_sop_json(filename):
#     cwd = os.getcwd()
#     os.chdir(notebook_dir)
#     with open(f'../data/interim/{filename}') as f:
#         one_type = json.load(f)
#     f.close()
#     os.chdir(cwd)
#     return one_type

# +
# juris_all, roles_all, sops_all = list(), list(), list()
# for filename in all_sop:
#     one_type = load_sop_json(filename)
#     for juri, dct in one_type.items():
#         for role, sop in dct.items():
#             juris_all.append(juri)
#             roles_all.append(role)
#             sops_all.append(sop)
# alldf = pd.DataFrame({'juri': juris_all, 'role': roles_all, 'sop': sops_all})
# alldf

# +
# alldf.iloc[0, :]['sop']

# +
# dt_matrix_all, corpus_all, dictionary_all = get_dct_dtmatrix(sops_all)

# +
# all_coherence = topics_with_coherence(dt_matrix_all, corpus_all, dictionary_all, N = 20)

# +
# all_coherence

# +
# plt.figure(figsize = (12, 8))
# plt.plot(all_coherence.loc[:, 'num_topic'].values, all_coherence.loc[:, 'coherence_score'].values)
# plt.show()
# -



StarcoderdataJupyterScriptsDedupFiltered