<jupyter_start><jupyter_text>1. Preprocess<jupyter_code>def preprocess(df, col):
    preprocessor = PreProcess()
    preprocessor.fill_na(df, col)
    preprocessor.remove_urls(df, col)
    preprocessor.expand_contractions(df, col)
    preprocessor.remove_escape_chars(df, col)
    return df<jupyter_output><empty_output><jupyter_text>Posts<jupyter_code>world_posts.head()
posts_df = world_posts.drop(columns=["subreddit", "url","title", "created"])
posts_df.head()
# posts_pre_df = PreProcess.preprocess(posts_df, 'body')
# for i in range(len(posts_df.index)):
#     posts_pre_df.at[i, 'body_string'] = ' '.join([str(elem) for elem in posts_pre_df.at[i, 'body_stem']])
# posts_pre_df.head()
preprocess(posts_df, "body")<jupyter_output><empty_output><jupyter_text>Comments<jupyter_code>comments_df = world_comments.copy()
comments_df.head()
preprocess(comments_df, "comment")<jupyter_output><empty_output><jupyter_text>2. BERT Sentiment Analysis<jupyter_code>tokenizer = AutoTokenizer.from_pretrained('nlptown/bert-base-multilingual-uncased-sentiment')
model = AutoModelForSequenceClassification.from_pretrained('nlptown/bert-base-multilingual-uncased-sentiment')
def sentiment_score(post):
    tokens = tokenizer.encode(post, return_tensors='pt')
    result = model(tokens)
    return int(torch.argmax(result.logits))+1
posts_df['sentiment'] = posts_df['body'].apply(lambda x: sentiment_score(x[:512]))
posts_df.head()
# comments_df['sentiment'] = comments_df['comment'].apply(lambda x: sentiment_score(x[:512]))
# comments_df.head()<jupyter_output><empty_output><jupyter_text>3. EDA sentiments of Posts and Comments<jupyter_code>posts_df.corr()
posts_df.sentiment.value_counts().loc[[1, 2, 3, 4, 5]].plot(kind='bar', xlabel='sentiment', ylabel='frequency')
pprint(posts_df.sentiment.value_counts().loc[[1, 2, 3, 4, 5]])
plt.show()
temp_id = posts_df.post_id[0]
print("Looking at one post with id:",temp_id)
posts_df.head(1)
median_sentiment_list=[]
for i in posts_df.post_id:
    median_sentiment_list.append(np.median(comments_df[comments_df.post_id==i].sentiment))
posts_df['median_comments_sentiment'] = median_sentiment_list
posts_df.head()
posts_df
posts_df.sentiment.value_counts().loc[[1, 2, 3, 4, 5]].plot(kind='bar', xlabel='sentiment', ylabel='frequency')
plt.show()
posts_df.median_comments_sentiment.value_counts().loc[[1, 2, 3, 4, 5]].plot(kind='bar', xlabel='median_sentiment', ylabel='frequency')
plt.show()<jupyter_output><empty_output><jupyter_text>4. Observing with post Metadata<jupyter_code>df = posts_df[['post_id','score','upvote_ratio','num_comments', 'sentiment']]
df
plt.rcParams["figure.figsize"] = (20,10)

plt.scatter(df.score, df.num_comments, c=df.sentiment, cmap="Greys")
plt.legend()
plt.show()
from sklearn.preprocessing import MinMaxScaler
df = posts_df[['post_id','score','upvote_ratio','num_comments', 'sentiment']]
sc = MinMaxScaler()
df = sc.fit_transform(df[['score','upvote_ratio','num_comments']])
df
from mpl_toolkits import mplot3d
from mpl_toolkits.mplot3d import Axes3D
plt.rcParams["figure.figsize"] = (20,15)

fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
# ax.scatter(df['score'], df['upvote_ratio'], df['num_comments'], c=df['sentiment'])
ax.scatter(df[:, 0], df[:, 1], df[:, 2], c=posts_df.sentiment)
plt.show()
#also try with another subreddit<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup