<jupyter_start><jupyter_text><jupyter_code>!curl -O https://raw.githubusercontent.com/franciscadias/data/master/abcnews-date-text.csv
import pandas as pd
df_data = pd.read_csv('./abcnews-date-text.csv')
df_data.head(3)
df_data = df_data.head(10000)
head_text = df_data[['headline_text']]
type(head_text)
import nltk
nltk.download('punkt')
head_text['title_text'] = head_text.apply(lambda row : nltk.word_tokenize(row['headline_text']), axis=1)  # map()
head_text.head(3)
from nltk.corpus import stopwords
nltk.download('stopwords')
stop = stopwords.words('english')
stop
# titles = []
# for x in head_text['title_text']:
#   for word in x:
#     if len(word) > 3:
#       if word not in stop:
#        titles.apply(word)
#   return titles

# def stopWord(x):
#   result = []
#   for word in x:
#     if word not in stop or len(word)>3:
#   return result

# def callStopWord(head_text):
#   titles=[]
#   for x in head_text['title_text']:
#     titles.apply(stopWord(x))
#   return titles

head_text['title'] = head_text['title_text'].apply(lambda x:[word for word in x if (len(word)>3) if (word not in stop)])
head_text.head(5)
head_text['title'][3]
tokens = []
for i in range(len(head_text)):
  tokens.append(' '.join(head_text['title'][i]))

tokens[3:5]
from sklearn.feature_extraction.text import TfidfVectorizer
tfidf = TfidfVectorizer(max_features=1000)
X = tfidf.fit_transform(tokens)
X.shape
X[4].toarray()
from sklearn.decomposition import LatentDirichletAllocation
lda_model = LatentDirichletAllocation()
lda_top = lda_model.fit_transform(X)
lda_model.components_.shape, lda_model.components_
terms = tfidf.get_feature_names()
n = 5
for idx, topic in enumerate(lda_model.components_):
  print([(terms[i], topic[i]) for i in topic.argsort()[:-n-1:-1]])<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup