# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import numpy as np
import pandas as pd
import re, spacy, gensim

# Sklearn
from sklearn.decomposition import LatentDirichletAllocation
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import GridSearchCV
from pprint import pprint

# Plotting tools
import pyLDAvis
import pyLDAvis.sklearn
# -

# Import Dataset
df = pd.read_json('https://raw.githubusercontent.com/selva86/datasets/master/newsgroups.json')
print(df.target_names.unique())

df.head(15)

data = df.content.values.tolist()[0:100]
# Remove Emails
data = [re.sub('\S*@\S*\s?', '', sent) for sent in data]
# Remove new line characters
data = [re.sub('\s+', ' ', sent) for sent in data]
# Remove distracting single quotes
data = [re.sub("\'", "", sent) for sent in data]
pprint(data[:1])


# +
def sent_to_words(sentences):
    for sentence in sentences:
        yield(gensim.utils.simple_preprocess(str(sentence), deacc=True))  # deacc=True removes punctuations

data_words = list(sent_to_words(data))

print(data_words[:1])


# +
def lemmatization(texts, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV']):
    """https://spacy.io/api/annotation"""
    texts_out = []
    for sent in texts:
        doc = nlp(" ".join(sent)) 
        texts_out.append(" ".join([token.lemma_ if token.lemma_ not in ['-PRON-'] else '' for token in doc if token.pos_ in allowed_postags]))
    return texts_out

# Initialize spacy 'en' model, keeping only tagger component (for efficiency)
# Run in terminal: python3 -m spacy download en
nlp = spacy.load('en', disable=['parser', 'ner'])

# Do lemmatization keeping only Noun, Adj, Verb, Adverb
data_lemmatized = lemmatization(data_words, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV'])

print(data_lemmatized[:2])

# +
vectorizer = CountVectorizer(analyzer='word',       
                             min_df=10,                        # minimum reqd occurences of a word 
                             stop_words='english',             # remove stop words
                             lowercase=True,                   # convert all words to lowercase
                             token_pattern='[<KEY>3,}',  # num chars > 3
                             # max_features=50000,             # max number of uniq words
                            )

data_vectorized = vectorizer.fit_transform(data_lemmatized)
# -

# Materialize the sparse data
data_dense = data_vectorized.todense()
# Compute Sparsicity = Percentage of Non-Zero cells
print("Sparsicity: ", ((data_dense > 0).sum()/data_dense.size)*100, "%")

# +
# Build LDA Model
lda_model = LatentDirichletAllocation(n_components=20,               # Number of topics
                                      max_iter=10,               # Max learning iterations
                                      learning_method='online',   
                                      random_state=100,          # Random state
                                      batch_size=128,            # n docs in each learning iter
                                      evaluate_every = -1,       # compute perplexity every n iters, default: Don't
                                      n_jobs = -1,               # Use all available CPUs
                                     )
lda_output = lda_model.fit_transform(data_vectorized)

print(lda_model)  # Model attributes

# +
# Log Likelyhood: Higher the better
print("Log Likelihood: ", lda_model.score(data_vectorized))

# Perplexity: Lower the better. Perplexity = exp(-1. * log-likelihood per word)
print("Perplexity: ", lda_model.perplexity(data_vectorized))

# See model parameters
pprint(lda_model.get_params())

# +
# Define Search Param
search_params = {'n_components': [10], 'learning_decay': [.5]}

# Init the Model
lda = LatentDirichletAllocation()

# Init Grid Search Class
model = GridSearchCV(lda, param_grid=search_params)

# Do the Grid Search
model.fit(data_vectorized)

# +
# Best Model
best_lda_model = model.best_estimator_

# Model Parameters
print("Best Model's Params: ", model.best_params_)

# Log Likelihood Score
print("Best Log Likelihood Score: ", model.best_score_)

# Perplexity
print("Model Perplexity: ", best_lda_model.perplexity(data_vectorized))

# +
# Create Document - Topic Matrix
lda_output = best_lda_model.transform(data_vectorized)

# column names
topicnames = ["Topic" + str(i) for i in range(best_lda_model.n_components)]

# index names
docnames = ["Doc" + str(i) for i in range(len(data))]

# Make the pandas dataframe
df_document_topic = pd.DataFrame(lda_output, columns=topicnames, index=docnames)

# Get dominant topic for each document
dominant_topic = np.argmax(df_document_topic.values, axis=1)
df_document_topic['dominant_topic'] = dominant_topic

df_document_topic.round(2)
# -

df_topic_distribution = df_document_topic['dominant_topic'].value_counts().reset_index(name="Num Documents")
df_topic_distribution.columns = ['Topic Num', 'Num Documents']
df_topic_distribution

pyLDAvis.enable_notebook()
panel = pyLDAvis.sklearn.prepare(best_lda_model, data_vectorized, vectorizer, mds='tsne')
panel

# +
# Topic-Keyword Matrix
df_topic_keywords = pd.DataFrame(best_lda_model.components_)

# Assign Column and Index
df_topic_keywords.columns = vectorizer.get_feature_names()
df_topic_keywords.index = topicnames

# View
df_topic_keywords.head()
# -

df_topic_keywords.sum(axis = 1)


# +
# Show top n keywords for each topic
def show_topics(vectorizer=vectorizer, lda_model=lda_model, n_words=20):
    keywords = np.array(vectorizer.get_feature_names())
    topic_keywords = []
    for topic_weights in lda_model.components_:
        top_keyword_locs = (-topic_weights).argsort()[:n_words]
        topic_keywords.append(keywords.take(top_keyword_locs))
    return topic_keywords

topic_keywords = show_topics(vectorizer=vectorizer, lda_model=best_lda_model, n_words=15)        

# Topic - Keywords Dataframe
df_topic_keywords = pd.DataFrame(topic_keywords)
df_topic_keywords.columns = ['Word '+str(i) for i in range(df_topic_keywords.shape[1])]
df_topic_keywords.index = ['Topic '+str(i) for i in range(df_topic_keywords.shape[0])]
df_topic_keywords
# -

ids = list(map(lambda x : int(x[3:]), list(df_document_topic[df_document_topic['dominant_topic']==4].index)))
data_to_summarize = [data[i] for i in ids]
len(data_to_summarize)

# +
from transformers import pipeline

def run_summary(gpt2_input):
    summarizer = pipeline("summarization")
    return summarizer(gpt2_input, max_length = 10)
test = """
<NAME> died years ago. But for devoted fans of the iconic Apple co-founder, it seems like just yesterday that Jobs stepped away from his role as Apple’s chief executive to seek treatment for the rare pancreatic cancer that would eventually kill him. But that begs the question: When did <NAME> die? And what were his last words?

Read on to get all the details on Jobs’ untimely death, what he regretted about the choices he made during his life, and what his sister said were <NAME>’ last words.
<NAME> developed a rare form of pancreatic cancer
<NAME> opens the Apple Worldwide Developers conference in 2005

<NAME> opened the Apple Worldwide Developers conference in 2005. | <NAME>/ Getty Images

WebMD reports that if <NAME> had developed the most common form of pancreatic cancer, adenocarcinoma, he would likely have died soon after his 2003 diagnosis. Instead, he had an unusual form of pancreatic cancer known as a neuroendocrine tumor or islet cell carcinoma, which typically has a much better prognosis. Islet cells — the hormone-producing cells of the pancreas — develop cancers that are highly treatable and often curable. In 2004, Jobs underwent surgery to remove the tumor.

Jobs is said to have undergone a Whipple procedure. That usually involves removing the head of the pancreas, part of the bile duct, the gallbladder, and the first part of the small intestine. Years later, in 2009, he received a liver transplant. That would have meant that cancer had spread to his liver. However, cancer can recur even after the transplant. And because the patient is on immune-suppressing anti-rejection drugs, there’s little that doctors can do. That’s what happened to <NAME>.
He regretted his treatment decisions

The Telegraph reports that <NAME> told his biographer that he regretted spending time trying to treat his cancer with alternative medicine. Jobs delayed operations and chemotherapy for nine months after his diagnosis in 2003 to try to treat his cancer without surgery. Biographer <NAME> explained, “I think he felt: if you ignore something you don’t want to exist, you can have magical thinking. It had worked for him in the past. He would regret it.”

Jobs’ wife, <NAME>, told Isaacson, “The big thing was he really was not ready to open his body. It’s hard to push someone to do that.” When Jobs did agree to traditional treatment, he had his DNA sequenced at the cost of $100,000. That enabled doctors to specifically target treatment to the particular molecular pathways that were defective in his body. But eventually, surgery revealed that cancer had spread beyond the pancreas to his liver.
But the tumor could have spread even with earlier treatment
<NAME> introduces the iPad 2

<NAME> waves to the crowd. | <NAME>/ Getty Images

<NAME> might have regretted the decision to try acupuncture, a vegan diet, herbs, and juices before operating on his cancer. But The New York Times notes that it’s impossible to know whether that would have stopped his cancer from eventually killing him. <NAME>, one of Jobs’ doctors, told the Times, “No one can say whether or not having surgery earlier would have made any difference because of the possibility of micrometastases.”

Micrometastases are tiny cancers that form in various organs when a tumor starts to spread around the body. The Times notes, “Dr. Ornish’s comment means that in theory, Mr. Jobs’ tumor could already have spread invisibly to his liver by the time it was first diagnosed. If it had, operating earlier probably would not have made a difference.” Another doctor explained that among patients with this kind of tumor, “when they are first found on a scan, about 60 percent of the time it’s already metastasized to the liver.”
When did <NAME> die?

Harvard Health reports that <NAME> died on October 5, 2011, “almost exactly eight years after his cancer was discovered incidentally on a CT scan of his kidneys (the pancreas is near the left kidney).” Jobs got the CT scan at the recommendation of his urologist, who was concerned about kidney stones he’d had several years earlier. Jobs’ liver transplant didn’t preclude his cancer from recurring. His liver was full of cancer when doctors removed it. That means that cancer had likely already spread outside the pancreas and liver at the time of the transplant.

When <NAME> died on October 5, 2011, he was just 56 years old. He had taken medical leave from Apple starting in January that year, CNN reports. And he stepped down as Apple’s chief executive in August 2011, saying he could “no longer meet (his) duties and expectations.” Jobs was survived by his wife of 20 years, Laurene, and four children, including one — <NAME>-Jobs — from a prior relationship.
"""
run_summary(test)

StarcoderdataJupyterScriptsDedupFiltered