# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

import os, sys
sys.path.insert(0, os.environ['PROJECT_PATH'])

# +
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer

from config.resources import path_to
# -

jbv_df = pd.read_csv(path_to['jbv_meta'])

# +
import matplotlib.pyplot as plt

# %matplotlib inline

# +
jbv_df['abstract_len'] = jbv_df['AB'].dropna().apply(lambda a: len(a.split()))

plt.plot(jbv_df['abstract_len'].tolist())

# +
abstracts = jbv_df['AB'].dropna()
nb_features = 600 

tfidf_vectorizer = TfidfVectorizer(max_df=0.95, min_df=2, max_features=nb_features, stop_words='english')
tfidf = tfidf_vectorizer.fit_transform(abstracts)
tfidf_feature_names = tfidf_vectorizer.get_feature_names()

# +
from sklearn.decomposition import NMF

nb_topics = 20

# Run NMF
nmf = NMF(n_components=nb_topics, random_state=1, alpha=.1, l1_ratio=.5, init='nndsvd').fit(tfidf)


# -

def display_topics(model, feature_names, no_top_words):
    for topic_idx, topic in enumerate(model.components_):
        print "Topic %d:" % (topic_idx)
        print " ".join([feature_names[i]
                        for i in topic.argsort()[:-no_top_words - 1:-1]])


nb_top_words = 10
display_topics(nmf, tf_feature_names, nb_top_words)



StarcoderdataJupyterScriptsDedupFiltered