# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
# ---

import pandas as pd
import numpy as np
# import pymssql
# from fuzzywuzzy import fuzz
import json
import tweepy
from collections import defaultdict
from datetime import datetime
import re
# import pyodbc
from wordcloud import WordCloud
import seaborn as sns
import matplotlib.pyplot as plt
from wordcloud import WordCloud
import string, nltk, re, json, tweepy, gensim, scipy.sparse, pickle, pyLDAvis, pyLDAvis.gensim
from sklearn.feature_extraction.text import CountVectorizer
from nltk.stem import WordNetLemmatizer
from nltk.corpus import stopwords
from gensim import matutils, models, corpora
import warnings
warnings.filterwarnings("ignore")

# # Topic Model

df = pd.read_csv('./meme_cleaning.csv')

df_sentiment = pd.read_csv('563_df_sentiments.csv')

df_sentiment = df_sentiment.drop(columns=['Unnamed: 0', 'Unnamed: 0.1', 'Unnamed: 0.1.1'])

#Extract all words that begin with # and turn the results into a dataframe
temp = df_sentiment['Tweet'].str.lower().str.extractall(r"(#\w+)")
temp.columns = ['unnamed']
# Convert the multiple hashtag values into a list
temp = temp.groupby(level = 0)['unnamed'].apply(list)
# Save the result as a feature in the original dataset
df_sentiment['hashtags'] = temp

# +
for i in range(len(df_sentiment)):
    if df_sentiment.loc[i, 'No_of_Retweets'] >= 4:
        df_sentiment.loc[i, 'No_of_Retweets'] = 4

for i in range(len(df_sentiment)):
    if df_sentiment.loc[i, 'No_of_Likes'] >= 10:
        df_sentiment.loc[i, 'No_of_Likes'] = 10
# -

retweet_df = df_sentiment.groupby(['No_of_Retweets', 'vaderSentiment']).vaderSentimentScores.agg(count='count').reset_index()
like_df = df_sentiment.groupby(['No_of_Likes', 'vaderSentiment']).vaderSentimentScores.agg(count='count').reset_index()
classify_df = df_sentiment.vaderSentiment.value_counts().reset_index()
df_sentiment.Labels = df_sentiment.Labels.fillna('')

df_likes_dict = df_sentiment.groupby('No_of_Likes').vaderSentimentScores.agg(count='count').to_dict()['count']
df_retweet_dict = df_sentiment.groupby('No_of_Retweets').vaderSentimentScores.agg(count='count').to_dict()['count']

# +
for i in range(len(like_df)):
  like_df.loc[i, 'Normalized_count'] = like_df.loc[i, 'count'] / df_likes_dict[like_df.loc[i, 'No_of_Likes']]

for i in range(len(retweet_df)):
  retweet_df.loc[i, 'Normalized_count'] = retweet_df.loc[i, 'count'] / df_retweet_dict[retweet_df.loc[i, 'No_of_Retweets']]
# -

cv = CountVectorizer(stop_words='english')
data_cv = cv.fit_transform(df.Tweet)
words = cv.get_feature_names()
data_dtm = pd.DataFrame(data_cv.toarray(), columns=cv.get_feature_names())
pickle.dump(cv, open("cv_stop.pkl", "wb"))

data_dtm_transpose = data_dtm.transpose()

sparse_counts = scipy.sparse.csr_matrix(data_dtm_transpose)
corpus = matutils.Sparse2Corpus(sparse_counts)

cv = pickle.load(open("cv_stop.pkl", "rb"))
id2word = dict((v, k) for k, v in cv.vocabulary_.items())
word2id = dict((k, v) for k, v in cv.vocabulary_.items())
d = corpora.Dictionary()
d.id2token = id2word
d.token2id = word2id

lda = gensim.models.ldamodel.LdaModel(corpus=corpus, id2word=id2word, num_topics=3, passes=10)

pyLDAvis.enable_notebook()
vis = pyLDAvis.gensim.prepare(lda, corpus, d)
vis

StarcoderdataJupyterScriptsDedupFiltered