# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + colab={"base_uri": "https://localhost:8080/"} id="qHtNatVwLr8_" outputId="8b98f820-a7b0-4c5b-accd-0756918a404d"
from google.colab import drive
drive.mount('/content/drive')

# + colab={"base_uri": "https://localhost:8080/"} id="0hS8lIdyMVZJ" outputId="8567ba1d-6db5-4568-b0bc-451ff4ea7dbc"
# %cd "/content/drive/MyDrive/COMP598-datascience"

# + id="UWH4AcGwKxFo"
import csv
import pandas as pd

# + id="tXE8ePXQLUp8"
ds = pd.read_csv("IRAhandle_tweets_1.csv")

# + colab={"base_uri": "https://localhost:8080/", "height": 564} id="F6w-Y97vLip2" outputId="cf5ddb18-9e73-4f59-89da-e3070cde084a"
ds.head()

# + colab={"base_uri": "https://localhost:8080/"} id="6v5iojhAMkrH" outputId="d271ee01-2dfd-4f3b-d555-11e87ebce01e"
ds.shape

# + id="ajFh0G5QNgm_"
ds = ds[:10000]

# + colab={"base_uri": "https://localhost:8080/"} id="p1pwuIURNltx" outputId="42412532-adea-4e91-900d-c9e86cfd4ccd"
ds.shape

# + colab={"base_uri": "https://localhost:8080/"} id="-nTuX5F4CGXV" outputId="e7b07351-5b35-4df6-ca42-ab4e603d084d"
ds['language'].unique()

# + id="oMYtvMcWNNCA"
## now we know there is only one spelling of "English" (and not "ENGLISH" or "english" etc)

# + id="axbyAy7J6j-R"
import re

# + id="8LHWcdEuNo_8"
temp_df = []
for idx in range(ds.shape[0]):
    ## if english and if no question mark
    if ds['language'][idx] == 'English' and re.search(r'\?', ds['content'][idx]) == None:
        temp_df.append(list(ds.iloc[idx]))
          
    

# + id="ZuAZVNjrPjjH"
df = pd.DataFrame(temp_df, columns=ds.columns)

# + colab={"base_uri": "https://localhost:8080/", "height": 564} id="gYmtUgVQPwr0" outputId="a3f6d5e7-a151-4768-fe18-4c747e3d092f"
df.head()

# + colab={"base_uri": "https://localhost:8080/"} id="rNaU6P0DPxHS" outputId="2cbe19ac-14cf-4ac7-94a1-d127339762df"
df.shape

# + colab={"base_uri": "https://localhost:8080/"} id="ImTi9N9F7VOm" outputId="e73a50fa-173f-4aa6-dd67-cf902db6bf0b"
df['post_type'].unique()

# + id="vbwgXPMF7Orq"
## WE HAVE MORE THAN WHAT WE WANT BECAUSE SOME ARE RETWEETS!!!!!!!!!!!
## Some people do not capitalise T
## Some people reply and do not mention trump


# + id="C23O6j0S75G8"
pattern = r'(^(Trump)|\W(Trump))\W' # string starts with 'Trump', or has not an alphanumeric followed by Trump, and is not followed by an alphanumeric
trump_mention = []

for idx in range(df.shape[0]):
    if re.search(pattern, df['content'][idx]):
        trump_mention.append("Y")
    else:
        trump_mention.append("N")
        
df['trump_mention'] = trump_mention

# + id="0B6H33ELS-Rw"
## IDENTIFYING REALDONALDTRUMP MENTION 

pattern = r'\@(realDonaldTrump)' # string containing the mention @realDonaldTrump
dtrump_mention = []

for idx in range(df.shape[0]):
    if re.search(pattern, df['content'][idx]):
        dtrump_mention.append("Y")
    else:
        dtrump_mention.append("N")
        
df['dtrump_mention'] = dtrump_mention

# + colab={"base_uri": "https://localhost:8080/", "height": 564} id="a0Ts211G799U" outputId="becb5047-516f-42ee-ee9b-445a0a4c68ef"
df.head()

# + id="TOkjnZKtBEfj"
##### 879 retweet or quote tweet
##### 590 retweet
##### 289 quote tweet

# + colab={"base_uri": "https://localhost:8080/"} id="LDeKRraJUlSi" outputId="d25386e7-82ec-4336-8c0e-4067f8234ca0"
## HERE WE COUNT IF REALDONALDTRUMP IS MENTIONED

rt_count = 0
qt_count = 0
count = 0
for idx in range(df.shape[0]):
    if df['dtrump_mention'][idx] == "Y":
        if df['post_type'][idx] == 'RETWEET':
            rt_count += 1
        elif df['post_type'][idx] == 'QUOTE_TWEET':
            qt_count += 1
        else:
            count += 1

print("Normal: {}\nRetweet: {}\nQuote tweet: {}\nTotal: {}".format(count, rt_count, qt_count, count+rt_count+qt_count))

# + colab={"base_uri": "https://localhost:8080/"} id="WRDL3Sge8Fee" outputId="4e0dbab8-03ab-439b-d74d-d72ab6d8640d"
## HERE WE COUNT IF REALDONALDTRUMP IS MENTIONED AND IF POST DOES NOT INCLUDE WORD "Trump"

rt_count = 0
qt_count = 0
count = 0
for idx in range(df.shape[0]):
    if df['dtrump_mention'][idx] == "Y" and df['trump_mention'][idx] == "N":
        if df['post_type'][idx] == 'RETWEET':
            rt_count += 1
        elif df['post_type'][idx] == 'QUOTE_TWEET':
            qt_count += 1
        else:
            count += 1

print("Normal: {}\nRetweet: {}\nQuote tweet: {}\nTotal: {}".format(count, rt_count, qt_count, count+rt_count+qt_count))

# + colab={"base_uri": "https://localhost:8080/"} id="oRCxci0VUUrw" outputId="e6fddf25-fbbd-4035-afc7-6a4416392a0d"
## HERE WE COUNT IN HOW MANY POSTS "Trump" was mentioned

rt_count = 0
qt_count = 0
count = 0
for idx in range(df.shape[0]):
    if df['trump_mention'][idx] == "Y":
        if df['post_type'][idx] == 'RETWEET':
            rt_count += 1
        elif df['post_type'][idx] == 'QUOTE_TWEET':
            qt_count += 1
        else:
            count += 1
trump_mentions = count+rt_count+qt_count
print("Normal: {}\nRetweet: {}\nQuote tweet: {}\nTotal: {}".format(count, rt_count, qt_count, trump_mentions))

# + id="y5AjS_7cxciQ"
final_df = pd.DataFrame(data= {'tweet_id': df['tweet_id'], 'publish_date': df['publish_date'], 'content': df['content'], 'trump_mention': df['trump_mention']})

# + colab={"base_uri": "https://localhost:8080/", "height": 204} id="RemD6nVN93aS" outputId="69a82001-8346-42c5-ef45-b794843569c9"
final_df.head()

# + id="Dps5l2FkxJ1j"
## tweet_id,publish_date,content,and trump_mention

# + colab={"base_uri": "https://localhost:8080/", "height": 204} id="QVOVyT0T7WWt" outputId="e9fbe8d8-7aa3-453c-b8fc-3d4c92bd2324"
final_df.head()

# + id="-lEdVr89vJbb"
final_df.to_csv('/content/drive/MyDrive/COMP598-datascience/dataset.tsv', sep='\t', index_label=False)

# + colab={"base_uri": "https://localhost:8080/"} id="8VrxKdmHKEjW" outputId="4b611a54-2a3a-42f9-a5ce-8f349ca8d244"
frac = trump_mentions / (final_df.shape[0])
print(frac)
print(round(frac, 3))

# + id="TUVOZxTminfT"
results_df = pd.DataFrame(data= {'result': ['frac-trump-mentions'], 'value': [round(frac, 3)]})

# + colab={"base_uri": "https://localhost:8080/", "height": 80} id="G1kZlkkmjl5r" outputId="593c3ff6-9930-498f-c186-4729f53c70d8"
results_df

# + id="yWNzwUQIlU72"
results_df.to_csv('/content/drive/MyDrive/COMP598-datascience/results.tsv', sep='\t', index=False)

# + id="sAtD3Y4zmoot"


StarcoderdataJupyterScriptsDedupFiltered