# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Demonstration Notebook: LDA Visualization

# ## Imports and Setup
#
# We first need to import the packages that we are going to for loading, processing and analysing the data.

# +
# the data is stored in a json file, so we use Python's json package to read it
import json

# regex
import re

# spacy is used for preprocessing
import spacy

# gensim is used to train a topic model
from gensim import models, corpora
from gensim.parsing.preprocessing import preprocess_documents, preprocess_string, strip_tags, strip_punctuation, strip_numeric
from gensim.models import CoherenceModel

# pyLDAvis used for the visualisation
import pyLDAvis
import pyLDAvis.gensim_models as pylda_gensimvis

# a progress bar is really helpful
from tqdm import tqdm

import pandas as pd
import seaborn as sns
import matplotlib.colors as mcolors
from wordcloud import WordCloud
import matplotlib.pyplot as plt

# also set up logging output
import logging
logging.basicConfig(format="%(asctime)s %(message)s", level=logging.INFO)

import pathlib

logger = logging.getLogger(__name__)
# -

# ## Data Loading
#
# We first load the data from the JSON file. For this experiment, we are only interested in the lyrics and discard all the metadata.

# +
# load data
path_to_corpus = "../data/rebetiko_corpus.json"
output_path = "../outputs/plots/"

output_path = pathlib.Path(output_path)
output_path.mkdir(exist_ok=True, parents=True)

with open(path_to_corpus) as f:
    corpus_data = json.load(f)

corpus_data = corpus_data["RECORDS"]

list_of_songs = []

for song in corpus_data:
    if song["lyrics"] is None:
        continue
    lyrics = song["lyrics"]
    lyrics = lyrics.replace("\n", " ")
    lyrics = re.sub(' +', ' ', lyrics)
    list_of_songs.append(lyrics.lower())
# -

# ## Preprocess Data
#
#
# For data preprocessing, we use spacy. First we load a language model. The processing pipeline consists of tokenization, tagging, parsing, named entity recognition and lemmatization. In addition, we add a component that removes stopwords.

# +
# load the spacy language model for the Greek language
nlp = spacy.load("el_core_news_lg")

# Get default stop words from spacy
default_stop_words = nlp.Defaults.stop_words

# Define additional stop words
additional_stopwords = ["μου","σου","σ\'","κι","απ\'","μ\'", "\'μ\'", "ν\'","του","σ", "γι\'", "δώσ\'",
"θά","μες","σαι","βρεις", "-","σένανε" ,"διάφορα","σύνθημα", "βλέπω","δεις", "\'ρθει", "κάν\'", "λεν\'", "σέ","τήν",
"πάω","τ","μ","εσύ","εσυ","νά","\'χα","την","και","τη","είπες", "ς","μια","συ", "\'συ", "τ\'", "είν\'", "γιά", "καί",
"ένα","τι","της","σένα","λες","\'χεις","πες","κάνω","κάνει","κάνεις","έχεις", "έχω","\'χω","στ\'", "είδ\'", "τόν", "μέ",
"π\'","θε","θες","μες","είδα","είδες","πήρες","βρω","ναι","είν","λέω","πεις", "απ","δως","τ", "\'χε", "\'ναι", "μεσ\'",
"ρθω","πάρει","σα","ε","ξέρεις","ξέρω","δω","άλλονε","ό","θ\'","έκανες", "πάει", "έχε", "είμ\'", "τό", "μπορώ",
"θέλει","ποιος","πει","\'γω","ρθει","πάρε","πάρω","μπορώ","πας","λες","λένε", "λενε", "πάν\'", "\'χει",
"βρήκα","καμιά","μένα","κάνε","γίνω","λέγω","έλεγα","είπα","τί","τι","\'πα", "βρω", "\'ν"]

# Add them to spacy's list of stop words
nlp.Defaults.stop_words.update(additional_stopwords)

# Iterates over the words in the stop words list and resets the "is_stop" flag.
for word in additional_stopwords:
    lexeme = nlp.vocab[word]
    lexeme.is_stop = True

docs = nlp.pipe(list_of_songs)

# process the corpus; the result is a list of lists of strings (i.e. a list of songs)
lyrics_for_lda = []

for doc in docs:
    lyrics_list = []

    tokens_filtered = [token.text for token in doc if not token.is_stop and token.is_alpha]
    lyrics_for_lda.append(tokens_filtered)
# -

# ## Model Training
#
# Now we can train a topic model via our preprocessed data. Here we train a Latent Dirichlet Allocation (LDA) model with the help of the gensim library.

# +
# create a dictionary representation of the documents
dictionary = corpora.Dictionary(lyrics_for_lda)

# filter words from the dictionary that are very rare or occur very often
dictionary.filter_extremes(no_below=3, no_above=0.6)

# compute a bag-of-words representation of the documents
corpus = [dictionary.doc2bow(doc) for doc in lyrics_for_lda]
# -

# now we can train a topic model
model = models.ldamodel.LdaModel(corpus, num_topics=5, id2word=dictionary, passes=15,eval_every=None)

# ## Visualising Results
#
# Now we can have a look into the trained model. We could use textual output for that, but a graphical visualisation is much more helpful. For this, we use the pyldavis package.

# +
# textual output of the identified topics
topics = model.print_topics(num_words=10)
for topic in topics:
    print(topic)

# visualisation
pyLDAvis.enable_notebook()
# for projecting the topics into a 2d space, we use tsne
vis = pylda_gensimvis.prepare(model, corpus, dictionary, mds='tsne')
#open the visualisation
vis

# +
#export pyLDAvis graphs as standalone webpage
#pyLDAvis.save_html(vis, output_path / 'lda.html')
# -

# ## Analyse the Created Topic Model

# +
# Compute Coherence Score
coherence_model_lda = CoherenceModel(model=model, texts=lyrics_for_lda, dictionary=dictionary, coherence='c_v')
coherence_lda = coherence_model_lda.get_coherence()

print('\nCoherence Score: ', coherence_lda)

for index, topic in model.show_topics(formatted=False, num_words=30):
    print('Topic: {} \nWords: {}'.format(index, [w[0] for w in topic]))

# +
lda_topics = model.show_topics(num_words=5)

topics = []
filters = [lambda x: x.lower(), strip_punctuation, strip_numeric]

for topic in lda_topics:
    topics.append(preprocess_string(topic[1], filters))


#compute coherence measure per topic and visualise it with seaborn
coherence_per_topic = coherence_model_lda.get_coherence_per_topic()

topics_str = [ '\n '.join(t) for t in topics ]
data_topic_score = pd.DataFrame( data=zip(topics_str, coherence_per_topic), columns=['Topic', 'Coherence'] )
data_topic_score = data_topic_score.set_index('Topic')

fig, ax = plt.subplots( figsize=(2,6) )
ax.set_title("Topics coherence\n $C_v$")
sns.heatmap(data=data_topic_score, annot=True, square=True,
            cmap='Reds', fmt='.2f',
            linecolor='black', ax=ax )
plt.yticks( rotation=0 )
ax.set_xlabel('')
ax.set_ylabel('')
fig.show()
# -

# ## Visualise the Topics in a Different Way

# +
cols = [color for name, color in mcolors.TABLEAU_COLORS.items()]  # more colors: 'mcolors.XKCD_COLORS'

cloud = WordCloud(background_color='white',
                  width=2500,
                  height=1800,
                  max_words=20,
                  colormap='tab10',
                  color_func=lambda *args, **kwargs: cols[i],
                  prefer_horizontal=1.0)

topics = model.show_topics(formatted=False)

fig, axes = plt.subplots(5, 1, figsize=(15,15), sharex=True, sharey=True)

for i, ax in enumerate(axes.flatten()):
    fig.add_subplot(ax)
    topic_words = dict(topics[i][1])
    cloud.generate_from_frequencies(topic_words, max_font_size=300)
    plt.gca().imshow(cloud)
    plt.gca().set_title('Topic ' + str(i), fontdict=dict(size=20))
    plt.gca().axis('off')
# -



StarcoderdataJupyterScriptsDedupFiltered