# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# Prerequisite : Need to run the NLP model notebook 

# This plot shows the word count before preprocess

# +
token_before_stop_words = reviews.apply(normalize)

fig = plt.figure(figsize = (10,4))
plt.gcf().subplots_adjust(bottom=0.15)
plt.tight_layout()
plt.title('Top 30 words in review before preprocessing')
plt.xlabel('Words')
plt.ylabel('Counts')


word_counts = list(itertools.chain(*token_before_stop_words))
freq_dist = FreqDist(word_counts)
freq_dist.plot(30, cumulative=False)

plt.show()
fig.savefig('../word_count_before_preprocess.png', bbox_inches = "tight")
# -

# This plot for review length will be used in the report. However, for the purpose of presentation, I will remove the outliers.

# +
import seaborn as sns 
processed_df['Review Length'] = processed_df["Tokens"].apply(lambda x: len(x))

processed_df.tail()


fig = plt.figure(figsize = (25,4))
plt.tight_layout()
plt.legend()
sns.countplot(processed_df['Review Length'])

plt.xticks(rotation = 90)

# -

# This plot will be used for the presentation purpose in which the outliers will be removed.
#
# IQR = Q3 - Q1 = 26-6 = 20 Upper_Bound = 26+1.5*20 = 56 Those review that have review length>56 will be removed from the plots for presentation.

# +
import seaborn as sns 
processed_df['Review Length'] = processed_df["Tokens"].apply(lambda x: len(x))

processed_df.tail()
df['Review Length'].describe()

fig = plt.figure(figsize = (10,4))
plt.tight_layout()
plt.legend()
sns.countplot(processed_df[processed_df['Review Length']<56]['Review Length'])
plt.xticks(rotation = 90)
plt.title('Review Length Distribution')

fig.savefig('../ReviewLength.png', bbox_inches = "tight")
# -

# From here, we can see that people mostly like to make short review. However, it might be due to the fact that our preprocessing 
# was unable to separate some words. Thus, there might be more words with a longer actual word length. 

# This plot shows the word count after preprocess

# +
fig = plt.figure(figsize = (10,4))
plt.tight_layout()
plt.title('Top 30 words in review after preprocessing')
plt.xlabel('Words')
plt.ylabel('Counts')


word_counts = list(itertools.chain(*tokens))
freq_dist = FreqDist(word_counts)
freq_dist.plot(30, cumulative=False)

plt.show()
fig.savefig('../freqDist.png', bbox_inches = "tight")
# -

# For this part of the code, I edited a bit to get the csv with the compound scores inside. 

# +
analyzer = SentimentIntensityAnalyzer()
def sentiment_polarity(s):
    global analyzer
    polarity_scores = analyzer.polarity_scores(s)
    compound_score = polarity_scores["compound"]
    if compound_score >= 0.5:
        label = "Positive"
    elif compound_score > -0.05 and compound_score < 0.05:
        label = "Neutral"
    else:
        label = "Negative"
    return label, polarity_scores["neu"], polarity_scores["pos"], polarity_scores["neg"],compound_score

df = processed_df
df["Sentiment"], df["Neutral Proportion"], df["Positive Proportion"], df["Negative Proportion"],df['Compound'] =  zip(*df["Review"].apply(sentiment_polarity))
df.sample(3)

df.to_csv("../Project/compound.csv")
# -

# The plots made by using the compound csv can be found in visualization_project_V3.rmd

StarcoderdataJupyterScriptsDedupFiltered