# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="XaQD4yBX7jO2"
# ## Loading the dataset

# + id="KyfTCPub61-p"
# Import PyDrive and associated libraries
# This only needs to be done once per notebook
from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
from google.colab import auth
from oauth2client.client import GoogleCredentials

# Authenticate and create the PyDrive client
# This only needs to be done once per notebook
auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
drive = GoogleDrive(gauth)

# + id="io0uGUvy7gWA"
# Download a file based on its file ID.

# A file ID looks like: laggVyWshwcyP6kEI-y_W3P8D26sz
file_id = '1QxkppL24m9TAT5Ix5AjJBvlK-6sfeN0V' # Check your own ID in GDrive
downloaded = drive.CreateFile({'id': file_id})

# Save file in Colab memory
downloaded.GetContentFile('tweet_data.csv')  

# + [markdown] id="OeQfMyVG7pU2"
# # Text Pre-Processing

# + id="bSYBXxKx7tH9" colab={"base_uri": "https://localhost:8080/"} outputId="9489d414-a644-4038-a346-957ea2a31ac7"
# !pip install tweet-preprocessor


# + id="iku57eXf7_u2"
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import preprocessor as p
import seaborn as sns

# + id="ylVp2Udw8TvG"
df = pd.read_csv('tweet_data.csv',lineterminator='\n')

# + id="Adbjg36k-UWV" colab={"base_uri": "https://localhost:8080/", "height": 1000} outputId="12d92123-6505-44c9-d528-61765c90bf11"
df

# + id="iuh_zy5l-bUp"
df = df[df["Language"] == "en"].drop(["Tweet Id","Unnamed: 0", "Language"], axis=1)


# + id="XW3an2d1-04h"
def clean_date(date_obj):
  for x in range(0, len(date_obj)):
    if date_obj[x] == " ":
      break
  date_obj = date_obj[:x]
  return date_obj

def preprocess_tweet(text):
    text = p.clean(text)
    return text



# + id="TpfXuc0xAEWk"
df["Datetime"] = df["Datetime"].apply(clean_date)
df['Clean Text'] = df['Text'].apply(preprocess_tweet)
df['Clean Text']= df['Clean Text'].str.replace('[^\w\s]','')

# + colab={"base_uri": "https://localhost:8080/"} id="on0u3R5JALS3" outputId="e55e5b9d-39a6-44e5-eeb2-9c2e944ad1b0"
# Removing Stop Words
import nltk
from nltk.corpus import stopwords
nltk.download('stopwords')
stop = stopwords.words('english')

# + id="nGlhsQiPAL7c"
df['Clean Text']  = df['Clean Text'].apply(lambda x: " ".join(x.lower() for x in x.split() if x not in stop))

# + colab={"base_uri": "https://localhost:8080/", "height": 213} id="AP0fCbqcAOVe" outputId="bf795eba-dfb1-4485-81e2-a54fe1d78222"
pd.set_option('display.max_colwidth', 400)
df.sample(2)

# + [markdown] id="dq7KS0paJ-3w"
# # TextBlob Sentiment Analysis, plotting the polarity scores' distribution

# + id="QJWgnB0xXXRz"
from textblob import TextBlob
from wordcloud import WordCloud


# + id="c0gHth7zA-Ei"
df['TBScore'] = df['Clean Text'].apply(lambda x: TextBlob(x).sentiment.polarity)
# Set threshold to define neutral sentiment
neutral_thresh = 0.05

# Convert polarity score into sentiment categories
df['Sentiment'] = df['TBScore'].apply(lambda c: 'Positive' if c >= neutral_thresh else ('Negative' if c <= -(neutral_thresh) else 'Neutral'))

# + colab={"base_uri": "https://localhost:8080/", "height": 436} id="ab2tWl3XXjKT" outputId="b48a187e-c52a-4288-824e-653037b801e7"
fig = plt.figure(figsize=(10, 6))
df['TBScore'].hist()
plt.xlabel('Polarity Score', fontsize=18)
plt.ylabel('Frequency', fontsize=18)
plt.xticks(fontsize=16)
plt.yticks(fontsize=16)

# + [markdown] id="q4ygrjeVKE3r"
# ## Plotting Word Clouds

# + colab={"base_uri": "https://localhost:8080/"} id="ChEkS_bZXmXG" outputId="7c7298a1-6910-4f32-a05c-6489c7aef115"
import nltk
nltk.download('stopwords')
from nltk.corpus import stopwords
from wordcloud import STOPWORDS

# + colab={"base_uri": "https://localhost:8080/", "height": 198} id="Kuca6HlzXp9S" outputId="3ddfea20-e3b3-4144-d901-811a1fc6b52c"
pos_tweets=df[df["Sentiment"]=="Positive"]
txt=" ".join(tweet for tweet in pos_tweets["Clean Text"])
stop_words = ["amp"] + list(STOPWORDS)
wordcloud = WordCloud(collocations = False,
                      background_color = 'white', stopwords = stop_words).generate(txt)
plt.imshow(wordcloud, interpolation="bilinear")
plt.axis("off")
plt.show()

# + colab={"base_uri": "https://localhost:8080/", "height": 198} id="2zF2hQhbXuzR" outputId="b1064df3-b9dd-4ab2-e6a8-21b9454099ec"
neg_tweets=df[df["Sentiment"]=="Negative"]
txt=" ".join(tweet.lower() for tweet in neg_tweets["Clean Text"])

wordcloud = WordCloud(collocations = False,background_color = 'white',stopwords=stop_words).generate(txt)
plt.imshow(wordcloud, interpolation="bilinear")
plt.axis("off")
plt.show()


# + [markdown] id="y8yPGSwAKHte"
# ## Plotting Bar Chart and Time-Series graphs

# + colab={"base_uri": "https://localhost:8080/", "height": 143} id="aKKM5FCxYCGb" outputId="8ef71e05-dbcd-4b82-f325-23adcf661695"
def get_value_counts(col_name):
    count = pd.DataFrame(df[col_name].value_counts())
    percentage = pd.DataFrame(df[col_name].value_counts(normalize=True).mul(100))
    value_counts_df = pd.concat([count, percentage], axis = 1)
    value_counts_df = value_counts_df.reset_index()
    value_counts_df.columns = ['sentiment', 'counts', 'percentage']
    value_counts_df.sort_values('sentiment', inplace = True)
    value_counts_df['percentage'] = value_counts_df['percentage'].apply(lambda x: round(x,2))
    value_counts_df = value_counts_df.reset_index(drop = True)
    return value_counts_df
tb_sentiment_df = get_value_counts('Sentiment')
tb_sentiment_df

# + colab={"base_uri": "https://localhost:8080/", "height": 295} id="e3Vu0sZpYJ_F" outputId="a8b7bc89-b137-4435-a96e-e70c359f9517"
ax = sns.barplot(x="sentiment", y="percentage", data=tb_sentiment_df)
ax.set_title('TextBlob results')

for index, row in tb_sentiment_df.iterrows():
    ax.text(row.name,row.percentage, round(row.percentage,1), color='black', ha="center")

# + colab={"base_uri": "https://localhost:8080/", "height": 523} id="0o5AsPNhYLWV" outputId="0db3821c-65de-4275-a249-6a110e66c88c"
# Plotting timeseries plot
df["Datetime"] = pd.to_datetime(df.Datetime) 
timeline = df.resample('D', on='Datetime')["Sentiment"].value_counts().unstack(1)
timeline.reset_index(inplace=True)
timeline = timeline.melt("Datetime", var_name='Sentiment',  value_name='Tweets')

plt.figure(figsize=(22,10))
ax = sns.lineplot(x="Datetime", y="Tweets", hue="Sentiment", data=timeline)
ax.figure.autofmt_xdate()

# + [markdown] id="wx6-ZYkyKSpu"
# ## Checking out the top 10 most Negative tweets (With Polarity of -1)

# + colab={"base_uri": "https://localhost:8080/", "height": 449} id="q0-rEcFvYO3V" outputId="b297c258-c40a-4e4a-f6ac-f2b56a54e506"
pd.set_option('display.max_colwidth', 400)
df.sort_values(by='TBScore', ascending=True)[['Text', 'TBScore', 'Username']].reset_index(drop=True).head(n=10)

# + [markdown] id="gnC2tPTRKYuW"
# ## Checking out the top 10 most Positive tweets (With Polarity of +1)

# + colab={"base_uri": "https://localhost:8080/", "height": 449} id="KOmeUXCrYR3H" outputId="80e4fd77-e008-458f-8361-5d8a10096127"
df.sort_values(by='TBScore', ascending=False)[['Text', 'TBScore', 'Username']].reset_index(drop=True).head(n=10)


# + [markdown] id="qDr2hT2mKq_7"
# # VADER (Valence Aware sEntiment Intensity Analyser) Sentiment Analysis

# + colab={"base_uri": "https://localhost:8080/"} id="6wwxilnIYTru" outputId="e0fa8200-3f1c-48c6-8098-cb38c5a58ce5"
# Performing VADER Sentiment Analysis
import nltk
nltk.download('vader_lexicon') # Download the VADER lexicon
from nltk.sentiment.vader import SentimentIntensityAnalyzer

# Initialize sentiment intensity analyzer
sia = SentimentIntensityAnalyzer()

# Obtaining NLTK scores
df['VScore'] = df['Clean Text'].apply(lambda x: sia.polarity_scores(x))

# Obtaining NLTK compound score
df['VComp'] = df['VScore'].apply(lambda score_dict: score_dict['compound'])

# Set threshold to define neutral sentiment
neutral_thresh = 0.05

# Categorize scores into the sentiments of positive, neutral or negative
df['Sentiment'] = df['VComp'].apply(lambda c: 'Positive' if c >= neutral_thresh else ('Negative' if c <= -(neutral_thresh) else 'Neutral'))


# + [markdown] id="MK2xeuBmK09n"
# ## Plotting the Polarity Scores' distribution, Bar Chart and Time-Series graphs

# + colab={"base_uri": "https://localhost:8080/", "height": 436} id="Xm8n4oC4ZF8J" outputId="a653fb69-c933-4118-d794-3871ef574233"
fig = plt.figure(figsize=(10, 6))
df['VComp'].hist()
plt.xlabel('Polarity Score', fontsize=18)
plt.ylabel('Frequency', fontsize=18)
plt.xticks(fontsize=16)
plt.yticks(fontsize=16)


# + colab={"base_uri": "https://localhost:8080/", "height": 143} id="q5sjpfOwZLcN" outputId="998884d9-0479-4d8f-b2e8-a6985bf94469"
def get_value_counts(col_name):
    count = pd.DataFrame(df[col_name].value_counts())
    percentage = pd.DataFrame(df[col_name].value_counts(normalize=True).mul(100))
    value_counts_df = pd.concat([count, percentage], axis = 1)
    value_counts_df = value_counts_df.reset_index()
    value_counts_df.columns = ['sentiment', 'counts', 'percentage']
    value_counts_df.sort_values('sentiment', inplace = True)
    value_counts_df['percentage'] = value_counts_df['percentage'].apply(lambda x: round(x,2))
    value_counts_df = value_counts_df.reset_index(drop = True)
    return value_counts_df
tb_sentiment_df = get_value_counts('Sentiment')
tb_sentiment_df

# + colab={"base_uri": "https://localhost:8080/", "height": 295} id="va9M-UlfZUti" outputId="257db838-5c84-4a06-acb4-e1fb88c77b13"
ax = sns.barplot(x="sentiment", y="percentage", data=tb_sentiment_df)
ax.set_title('Vader results')

for index, row in tb_sentiment_df.iterrows():
    ax.text(row.name,row.percentage, round(row.percentage,1), color='black', ha="center")

# + colab={"base_uri": "https://localhost:8080/", "height": 523} id="ZWvFUfRSZXEy" outputId="8bffcf87-147a-4e6e-8c90-de869053dd04"
df["Datetime"] = pd.to_datetime(df.Datetime) 
timeline = df.resample('D', on='Datetime')["Sentiment"].value_counts().unstack(1)
timeline.reset_index(inplace=True)
timeline = timeline.melt("Datetime", var_name='Sentiment',  value_name='Tweets')

plt.figure(figsize=(22,10))
ax = sns.lineplot(x="Datetime", y="Tweets", hue="Sentiment", data=timeline)
ax.figure.autofmt_xdate()

StarcoderdataJupyterScriptsDedupFiltered