# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Natural Language Analytics
#
# ## Exercise 3 - Topic Modeling and NER
#
# ### <NAME> - DSC18014

# ## Question A
#
# First let's load our libraries and our data

import pandas as pd
import glob
import os
from sklearn.model_selection import GridSearchCV
from sklearn.decomposition import NMF, LatentDirichletAllocation, TruncatedSVD
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.pyplot import figure
from sklearn.metrics.pairwise import euclidean_distances
import pyLDAvis.sklearn

# After we download we take a peek

subdir = os.listdir('nipstxt')
print(subdir)

# Let's exclude the files and folders that notes, indices and the orig folder

subdir = [ item for item in subdir if "nips" in item]
print(subdir)

lst_docs = []
for dirs in subdir:
   path = 'nipstxt/' + dirs + '/*.txt'
   files = glob.glob(path)
   for name in files:
       with open(name, encoding="utf8", errors='ignore') as f:
            lst_docs.append(f.read())

# Ok, now we have all our documents as items in a list.
# As a next step we will process the texts and transform the to vectors.
# We will experiment with computing both the inverse term frequency (TFIDF vectorizer)and Term frequency (sklearn's Count Vectorizer).

# #### TFIDF Vectorizer

# We choose to tokenize by english words that are 3 lettered and longer. We also consider to tokenizer by unigrams

extr = TfidfVectorizer(min_df=5, max_df=0.9, max_features=None, strip_accents='unicode', lowercase=True, analyzer='word',
                       token_pattern=r'\w{3,}', ngram_range=(1, 1), use_idf=True, smooth_idf=True,
                       sublinear_tf=True, stop_words="english")

# Let's vectorize our data

transformed_data = extr.fit_transform(lst_docs)

# Sklearn assumes alpha=1/n_components if no value is entered for doc_topic_prior. So in our exercise we will test for n_components 10 and 100. So our alpha will be 0.1 and 0.01

# +
NUM_TOPICS = 10
 
# Build a Latent Dirichlet Allocation Model
lda_model = LatentDirichletAllocation(n_components=NUM_TOPICS, max_iter=10, learning_method='online')
lda_Z = lda_model.fit_transform(transformed_data)
print(lda_Z.shape)  # (NO_DOCUMENTS, NO_TOPICS)
# -

# Let's see how the first document in the corpus looks like in different topic spaces

print(lda_Z[0])


# +
def print_topics(model, vectorizer, top_n=10):
    for idx, topic in enumerate(model.components_):
        print("Topic %d:" % (idx))
        print([(vectorizer.get_feature_names()[i], topic[i]) for i in topic.argsort()[:-top_n - 1:-1]])
        print("Most representative document: ", np.argmax(topic))

print_topics(lda_model, extr)
# -

# wow, only one describes best the tokens. Let's see the distribution of topics across the documents

win_topic=[np.argmax(topics) for topics in lda_Z]
plt.plot(win_topic)
plt.show()

# As expected... all documents could belong to topic 7

# Let's assume a new text and try to find the most similar document from our dataset

text = "the vlsi network makes no sense"
x = lda_model.transform(extr.transform([text]))[0]
print(x)


# +
def most_similar(x, Z, top_n=5):
    dists = euclidean_distances(x.reshape(1, -1), Z)
    pairs = enumerate(dists[0])
    most_similar = sorted(pairs, key=lambda item: item[1])[:top_n]
    return most_similar
 
similarities = most_similar(x, lda_Z)
document_id, similarity = similarities[0]
print(lst_docs[document_id][:1000])
# -

# To visualise the results of LDA let's import pyLDAvis

pyLDAvis.enable_notebook()
panel = pyLDAvis.sklearn.prepare(lda_model, transformed_data, extr, mds='tsne', sort_topics=False)
panel

# Well, one topic seems to dominate the majority of tokens. Here counting starts from 1, so it's the same as the previously found

# +
# Log Likelyhood: Higher the better
print("Log Likelihood: ", lda_model.score(transformed_data))

# Perplexity: Lower the better. Perplexity = exp(-1. * log-likelihood per word)
print("Perplexity: ", lda_model.perplexity(transformed_data))
# -

# Let's do a grid search

# +
# Define Search Param
search_params = {'doc_topic_prior':[.01,.1,.2,.5],'n_components': [5, 10, 50], 'learning_decay': [.2, .5, .7]}

# Init the Model
lda = LatentDirichletAllocation(max_iter=10, learning_method='online')

# Init Grid Search Class
model = GridSearchCV(lda, param_grid=search_params)

# Do the Grid Search
model.fit(transformed_data)

# Best Model
best_lda_model = model.best_estimator_

# Model Parameters
print("Best Model's Params: ", model.best_params_)

# Log Likelihood Score
print("Best Log Likelihood Score: ", model.best_score_)

# Perplexity
print("Model Perplexity: ", best_lda_model.perplexity(transformed_data))

lda_Z = best_lda_model.fit_transform(transformed_data)
#distribution
win_topic=[np.argmax(topics) for topics in lda_Z]
# -

from matplotlib.pyplot import figure
figure(num=None, figsize=(15, 6), dpi=80, facecolor='w', edgecolor='k')
plt.plot(win_topic)
plt.show()

print_topics(lda_model, extr)

# +
# Log Likelyhood: Higher the better
print("Log Likelihood: ", lda_model.score(transformed_data))

# Perplexity: Lower the better. Perplexity = exp(-1. * log-likelihood per word)
print("Perplexity: ", lda_model.perplexity(transformed_data))
# -

# Let's try to change the way we represent our features and see how LDA responds

# Again for a =0.1

# +
extr = CountVectorizer(min_df=5, max_df=0.9, stop_words='english', lowercase=True, token_pattern=r'\w{3,}')
transformed_data = extr.fit_transform(lst_docs)
NUM_TOPICS = 10

# Build a Latent Dirichlet Allocation Model
lda_model = LatentDirichletAllocation(n_components=NUM_TOPICS, max_iter=10, learning_method='online')
lda_Z = lda_model.fit_transform(transformed_data)
#distribution
win_topic=[np.argmax(topics) for topics in lda_Z]
# -

figure(num=None, figsize=(15, 6), dpi=80, facecolor='w', edgecolor='k')
plt.plot(win_topic)
plt.show()

print_topics(lda_model, extr)

# +
# Log Likelyhood: Higher the better
print("Log Likelihood: ", lda_model.score(transformed_data))

# Perplexity: Lower the better. Perplexity = exp(-1. * log-likelihood per word)
print("Perplexity: ", lda_model.perplexity(transformed_data))
# -

# pyLDAvis.enable_notebook()
panel = pyLDAvis.sklearn.prepare(lda_model, transformed_data, extr, mds='tsne', sort_topics=False)
panel

# With the Count Vectorizer we have a more meaningful result. Topic 7 is slightly more popular that the others.

# Let's do a grid search.

# +
# Define Search Param
search_params = {'doc_topic_prior':[.01,.1,.2,.5],'n_components': [5, 10, 50], 'learning_decay': [.2, .5, .7]}

# Init the Model
lda = LatentDirichletAllocation(max_iter=10, learning_method='online')

# Init Grid Search Class
model = GridSearchCV(lda, param_grid=search_params)

# Do the Grid Search
model.fit(transformed_data)

# Best Model
best_lda_model = model.best_estimator_

# Model Parameters
print("Best Model's Params: ", model.best_params_)

# Log Likelihood Score
print("Best Log Likelihood Score: ", model.best_score_)

# Perplexity
print("Model Perplexity: ", best_lda_model.perplexity(transformed_data))

lda_Z = best_lda_model.fit_transform(transformed_data)
#distribution
win_topic=[np.argmax(topics) for topics in lda_Z]

figure(num=None, figsize=(15, 6), dpi=80, facecolor='w', edgecolor='k')
plt.plot(win_topic)
plt.show()
# -

# ## Question B

# +
import collections
ner_tags = collections.Counter()
corpus_root = "gmb-2.2.0" 
 
for root, dirs, files in os.walk(corpus_root):
    for filename in files:
        if filename.endswith(".tags"):
            with open(os.path.join(root, filename), 'rb') as file_handle:
                file_content = file_handle.read().decode('utf-8').strip()
                annotated_sentences = file_content.split('\n\n')   # Split sentences
                for annotated_sentence in annotated_sentences:
                    annotated_tokens = [seq for seq in annotated_sentence.split('\n') if seq]  # Split words
 
                    standard_form_tokens = []
 
                    for idx, annotated_token in enumerate(annotated_tokens):
                        annotations = annotated_token.split('\t')   # Split annotation
                        word, tag, ner = annotations[0], annotations[1], annotations[3]
 
                        # Get only the primary category
                        if ner != 'O':
                            ner = ner.split('-')[0]
 
                        ner_tags[ner] += 1
print(ner_tags)
 
print("Words=", sum(ner_tags.values()))

# +
import string
from nltk.stem.snowball import SnowballStemmer

def features(tokens, index, history):
    """
    `tokens`  = a POS-tagged sentence [(w1, t1), ...]
    `index`   = the index of the token we want to extract features for
    `history` = the previous predicted IOB tags
    """
 
    # init the stemmer
    stemmer = SnowballStemmer('english')
 
    # Pad the sequence with placeholders
    tokens = [('[START2]', '[START2]'), ('[START1]', '[START1]')] + list(tokens) + [('[END1]', '[END1]'), ('[END2]', '[END2]')]
    history = ['[START2]', '[START1]'] + list(history)
 
    # shift the index with 2, to accommodate the padding
    index += 2
 
    word, pos = tokens[index]
    prevword, prevpos = tokens[index - 1]
    prevprevword, prevprevpos = tokens[index - 2]
    nextword, nextpos = tokens[index + 1]
    nextnextword, nextnextpos = tokens[index + 2]
    previob = history[index - 1]
    contains_dash = '-' in word
    contains_dot = '.' in word
    allascii = all([True for c in word if c in string.ascii_lowercase])
 
    allcaps = word == word.capitalize()
    capitalized = word[0] in string.ascii_uppercase
 
    prevallcaps = prevword == prevword.capitalize()
    prevcapitalized = prevword[0] in string.ascii_uppercase
 
    nextallcaps = prevword == prevword.capitalize()
    nextcapitalized = prevword[0] in string.ascii_uppercase
 
    return {
        'word': word,
        'lemma': stemmer.stem(word),
        'pos': pos,
        'all-ascii': allascii,
 
        'next-word': nextword,
        'next-lemma': stemmer.stem(nextword),
        'next-pos': nextpos,
 
        'next-next-word': nextnextword,
        'nextnextpos': nextnextpos,
 
        'prev-word': prevword,
        'prev-lemma': stemmer.stem(prevword),
        'prev-pos': prevpos,
 
        'prev-prev-word': prevprevword,
        'prev-prev-pos': prevprevpos,
 
        'prev-iob': previob,
 
        'contains-dash': contains_dash,
        'contains-dot': contains_dot,
 
        'all-caps': allcaps,
        'capitalized': capitalized,
 
        'prev-all-caps': prevallcaps,
        'prev-capitalized': prevcapitalized,
 
        'next-all-caps': nextallcaps,
        'next-capitalized': nextcapitalized,
    }


# +
def to_conll_iob(annotated_sentence):
    """
    `annotated_sentence` = list of triplets [(w1, t1, iob1), ...]
    Transform a pseudo-IOB notation: O, PERSON, PERSON, O, O, LOCATION, O
    to proper IOB notation: O, B-PERSON, I-PERSON, O, O, B-LOCATION, O
    """
    proper_iob_tokens = []
    for idx, annotated_token in enumerate(annotated_sentence):
        tag, word, ner = annotated_token
 
        if ner != 'O':
            if idx == 0:
                ner = "B-" + ner
            elif annotated_sentence[idx - 1][2] == ner:
                ner = "I-" + ner
            else:
                ner = "B-" + ner
        proper_iob_tokens.append((tag, word, ner))
    return proper_iob_tokens
 

def read_gmb(corpus_root):
    for root, dirs, files in os.walk(corpus_root):
        for filename in files:
            if filename.endswith(".tags"):
                with open(os.path.join(root, filename), 'rb') as file_handle:
                    file_content = file_handle.read().decode('utf-8').strip()
                    annotated_sentences = file_content.split('\n\n')
                    for annotated_sentence in annotated_sentences:
                        annotated_tokens = [seq for seq in annotated_sentence.split('\n') if seq]
 
                        standard_form_tokens = []
 
                        for idx, annotated_token in enumerate(annotated_tokens):
                            annotations = annotated_token.split('\t')
                            word, tag, ner = annotations[0], annotations[1], annotations[3]
 
                            if ner != 'O':
                                ner = ner.split('-')[0]
 
                            if tag in ('LQU', 'RQU'):   # Make it NLTK compatible
                                tag = "``"
 
                            standard_form_tokens.append((word, tag, ner))
 
                        conll_tokens = to_conll_iob(standard_form_tokens)
 
                        # Make it NLTK Classifier compatible - [(w1, t1, iob1), ...] to [((w1, t1), iob1), ...]
                        # Because the classfier expects a tuple as input, first item input, second the class
                        yield [((w, t), iob) for w, t, iob in conll_tokens]

reader = read_gmb(corpus_root)
# -

print (next(reader))

# +
import pickle
from collections import Iterable
from nltk.tag import ClassifierBasedTagger
from nltk.chunk import ChunkParserI, conlltags2tree, tree2conlltags
 

class NamedEntityChunker(ChunkParserI):
    def __init__(self, train_sents, **kwargs):
        assert isinstance(train_sents, Iterable)
 
        self.feature_detector = features
        self.tagger = ClassifierBasedTagger(
            train=train_sents,
            feature_detector=features,
            **kwargs)
 
    def parse(self, tagged_sent):
        chunks = self.tagger.tag(tagged_sent)
 
        # Transform the result from [((w1, t1), iob1), ...] 
        # to the preferred list of triplets format [(w1, t1, iob1), ...]
        iob_triplets = [(w, t, c) for ((w, t), c) in chunks]
 
        # Transform the list of triplets to nltk.Tree format
        return conlltags2tree(iob_triplets)


# +
reader = read_gmb(corpus_root)
data = list(reader)
training_samples = data[:int(len(data) * 0.9)]
test_samples = data[int(len(data) * 0.9):]
 
print ("#training samples = %s" % len(training_samples))
print ("#test samples = %s" % len(test_samples)  )
# -

chunker = NamedEntityChunker(training_samples[:2000])

from nltk import pos_tag, word_tokenize
print (chunker.parse(pos_tag(word_tokenize("I'm going to Germany this Monday."))))



StarcoderdataJupyterScriptsDedupFiltered