# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
# %matplotlib inline
import requests
import tweepy
import json
import time
import yaml
import os
import re

# # Gathering Data 

# ### 1. Twitter Archive 

twitter_archive_enhanced = pd.read_csv('twitter-archive-enhanced.csv')
twitter_archive_enhanced.head()

# ### 2. Image Predictions

# +
url = 'https://d17h27t6h515a5.cloudfront.net/topher/2017/August/599fd2ad_image-predictions/image-predictions.tsv'
response = requests.get(url)

with open('image_predictions.tsv', 'wb') as file:
    file.write(response.content)
    
image_predictions = pd.read_csv('image_predictions.tsv', sep='\t')
image_predictions.head()
# -

# ### 3. Twitter API & Twepy

# +
twitter_credentials = yaml.load(open('./twitter_credentials.yml'))

consumer_key = twitter_credentials['api']['api_consumer_key']
consumer_secret = twitter_credentials['api']['api_consumer_secret']
access_token = twitter_credentials['access']['access_token']
access_token_secret = twitter_credentials['access']['access_secret']

auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)

api = tweepy.API(auth, parser= tweepy.parsers.JSONParser(),
                 wait_on_rate_limit=True, wait_on_rate_limit_notify=True)
# -

# #### 3.1 Gathering 1

# +
missing_tweets = []

start_time = time.asctime()
start_sec = time.time()

with open ('tweet_json.txt', 'a') as file:
    for tweet_id in twitter_archive_enhanced.tweet_id:
        try:
            tweet = api.get_status(tweet_id, tweet_mode='extended')
            file.write(json.dumps(tweet) + '\n')
        except Exception as tweet_error_msg:
            missing_tweets.append(tweet_id)
    print('The End')
    
end_time = time.asctime()
end_sec = time.time()
# -

start_time , end_time

start_sec , end_sec

end_sec - start_sec

len(missing_tweets)

(end_sec - start_sec) / 60

# 29 mins and 25 seconds<br>
# 32 mins and 09 seconds

# ### 3.1 Tweet Json

# +
my_demo_list = []
with open('tweet_json.txt') as json_file:
    for line in json_file:
        each_dictionary = json.loads(line)
        tweet_id = each_dictionary['id']
        favorite_count = each_dictionary['favorite_count']
        retweet_count = each_dictionary['retweet_count']
        followers_count = each_dictionary['user']['followers_count']
        favourites_count = each_dictionary['user']['favourites_count']
        friends_count = each_dictionary['user']['friends_count']
        date_time = each_dictionary['created_at']

        my_demo_list.append({'tweet_id': int(tweet_id),
                             'favorite_count': int(favorite_count),
                             'retweet_count': int(retweet_count),
                             'followers_count': int(followers_count),
                             'favourites_count': int(favourites_count),
                             'friends_count': int(friends_count),
                             'date_time': pd.to_datetime(date_time)
                            })

tweet_json = pd.DataFrame(my_demo_list, columns = ['tweet_id', 'favorite_count','retweet_count',
                                              'followers_count', 'favourites_count',
                                              'friends_count', 'date_time'])
# -

# # Assessing visually

twitter_archive_enhanced

image_predictions

tweet_json

# # Programatic Assessment

twitter_archive_enhanced.info()

image_predictions.info()

tweet_json.info()

# ## Assessing: `Twitter Archive Enhanced Data`

twitter_archive_enhanced.head()

twitter_archive_enhanced.sample(5)

twitter_archive_enhanced.columns

# ######  Quality: 
# - `Source` format is bad and can't be read easily.
#
# ######  Tidiness: 
# - No need to divide Dog stages in 4 different columns like 
#     - `'doggo'`
#     - `'floofer'`
#     - `'pupper'`
#     - `'puppo'`

twitter_archive_enhanced.info()

# ######  Quality: 
#
# - These attributes should be `integers/strings` instead of `float`
#     - in_reply_to_status_id
#     - in_reply_to_user_id
#     - retweeted_status_id
#     - retweeted_status_user_id
# - Columns should be `datetime` instead of `object(string)`.
#     - retweeted_status_timestamp
#     - timestamp
# - We may want to change these columns types to `string` because We don't want any operations on them.
#     - tweet_id
#     - in_reply_to_status_id   
#     - in_reply_to_user_id 
#     - retweeted_status_id 
#     - retweeted_status_user_id
#

twitter_archive_enhanced[twitter_archive_enhanced.tweet_id.duplicated()]

twitter_archive_enhanced.describe()

twitter_archive_enhanced.describe(include=[object])

twitter_archive_enhanced.source.value_counts()

twitter_archive_enhanced[twitter_archive_enhanced.rating_numerator > 20]

# ###### Quality:
#
# - The columns `numerator` and `denominator` have invalid values.

np.sort(twitter_archive_enhanced.name.unique())

twitter_archive_enhanced.loc[(twitter_archive_enhanced['name'].str.islower())]

# ###### Quality:
#
# - There are invalid names like (`a, an, O, etc.`).

twitter_archive_enhanced[twitter_archive_enhanced.retweeted_status_id.isnull()]

# ###### Quality:
#
# - There are retweeted tweets, and we do not want it.





# ## Assessing: `Image Predictions Data`

image_predictions.head()

image_predictions.sample(5)

image_predictions.info()

# ###### Quality:
#
# - `tweet_id` should be `object` instead of `integer`, as no calculation is needed.

image_predictions.isnull().sum()

image_predictions.tweet_id.duplicated().sum()

image_predictions.describe()

image_predictions.describe(include='object')

image_predictions.jpg_url.value_counts()

# ###### Quality:
#
# - Missing values in `image_predictions` dataset. __2075__ rows of data compared to `twitter_archive_enhanced` dataset which has __2356__ number of records.
# - Some tweets have 2 different `tweet_id`, that are actually retweets.



# ## Assessing: `Tweets (API) JSON Data`

tweet_json.head()

tweet_json.sample(5)

tweet_json['tweet_id'].duplicated().sum()

tweet_json.info()

tweet_json.describe()

# ## Assessing Summary
#
# #### Quality
#
# - __`Twitter Archive Enhanced Data:`__ `Source` format is bad and can't be read easily.
# - __`Twitter Archive Enhanced Data:`__ These attributes should be `integers/strings` instead of `float`
#     - in_reply_to_status_id
#     - in_reply_to_user_id
#     - retweeted_status_id
#     - retweeted_status_user_id
# - __`Twitter Archive Enhanced Data:`__ Columns should be `datetime` instead of `object(string)`.
#     - retweeted_status_timestamp
#     - timestamp
# - __`Twitter Archive Enhanced Data:`__ We may want to change these columns types to `string` because We don't want any operations on them.
#     - tweet_id
#     - in_reply_to_status_id   
#     - in_reply_to_user_id 
#     - retweeted_status_id 
#     - retweeted_status_user_id
# - __`Twitter Archive Enhanced Data:`__ The columns `numerator` and `denominator` have invalid values.
# - __`Twitter Archive Enhanced Data:`__ Replacing all lower case invalid names like (`a, an, etc.`) with `None`
# - __`Twitter Archive Enhanced Data:`__ Replacing all `O` invalid names with `None`
# - __`Image Predictions Data:`__ `tweet_id` should be `object` instead of `integer`, as no calculation is needed.
# - __`Image Predictions Data:`__ Missing values in _`image_predictions`_ dataset. __2075__ rows of data compared to _`twitter_archive_enhanced`_ dataset which has __2356__ number of records.
# - __`Image Predictions Data:`__ Some tweets have 2 different `tweet_id`, that are actually retweets.
#
#
# #### Tidiness
#
# - __`Twitter Archive Enhanced Data:`__ No need to divide Dog stages in 4 different columns like 
#     - 'doggo'
#     - 'floofer'
#     - 'pupper'
#     - 'puppo'
# -  __`Image Predictions Data:`__  _`Image Predictions Data`_ should be joined to _`Twitter Archive Enhanced Data`_
# -  __`Tweets (API) JSON Data:`__ Merge _`Tweets (API) JSON Data`_ with the _`Twitter Archive Enhanced Data`_ 
#
# ---

# ## Cleaning Data
#
# Here we will fix the quality and tidiness issues that we identified earlier in the assessing step.
#
# #### Create Copy of Each Dataset(DataFrame)

twitter_archive_enhanced_clean = twitter_archive_enhanced.copy()
image_predictions_clean = image_predictions.copy()
tweet_json_clean = tweet_json.copy()

# ###### Issue: __`Tweets (API) JSON Data:`__ Merge _`Tweets (API) JSON Data`_ with the _`Twitter Archive Enhanced Data`_

# ###### Define: 

# Merge _`Tweets (API) JSON Data`_ with the _`Twitter Archive Enhanced Data`_ 

# ###### Code: 

twitter_archive_enhanced_clean = pd.merge(left=twitter_archive_enhanced_clean,
                                          right=tweet_json_clean,
                                          left_on='tweet_id',
                                          right_on='tweet_id',
                                          how='inner')

# ###### Test: 

twitter_archive_enhanced_clean.info()

# ###### Issue: __`Image Predictions Data:`__  _`Image Predictions Data`_ should be joined to _`Twitter Archive Enhanced Data`_

# ###### Define: 

# _`Image Predictions Data`_ should be joined to _`Twitter Archive Enhanced Data`_

# ###### Code: 

twitter_archive_enhanced_clean = twitter_archive_enhanced_clean.merge(image_predictions_clean,
                                                                     on='tweet_id',
                                                                     how='inner')

# ###### Test: 

twitter_archive_enhanced_clean.info()

# ###### Issue: __`Twitter Archive Enhanced Data:`__ No need to divide Dog stages in 4 different columns like
# - 'doggo'
# - 'floofer'
# - 'pupper'
# - 'puppo'

# ###### Define: 

# Combining the Dog stages to column `dog_stage` which are divided in 4 different columns like
# - 'doggo'
# - 'floofer'
# - 'pupper'
# - 'puppo'

# ###### Code: 

twitter_archive_enhanced_clean.loc[twitter_archive_enhanced_clean.doggo == 'None', 'doggo'] = ''
twitter_archive_enhanced_clean.loc[twitter_archive_enhanced_clean.floofer == 'None', 'floofer'] = ''
twitter_archive_enhanced_clean.loc[twitter_archive_enhanced_clean.pupper == 'None', 'pupper'] = ''
twitter_archive_enhanced_clean.loc[twitter_archive_enhanced_clean.puppo == 'None', 'puppo'] = ''

twitter_archive_enhanced_clean.groupby(["doggo", "floofer", "pupper", "puppo"]).size().reset_index().rename(columns={0: "count"})

# +
twitter_archive_enhanced_clean['dog_stage'] = twitter_archive_enhanced_clean.doggo + \
                                            twitter_archive_enhanced_clean.floofer + \
                                            twitter_archive_enhanced_clean.pupper + \
                                            twitter_archive_enhanced_clean.puppo

twitter_archive_enhanced_clean.loc[twitter_archive_enhanced_clean.dog_stage == 'doggopupper', 'dog_stage'] = 'doggo,pupper'
twitter_archive_enhanced_clean.loc[twitter_archive_enhanced_clean.dog_stage == 'doggopuppo', 'dog_stage'] = 'doggo,puppo'
twitter_archive_enhanced_clean.loc[twitter_archive_enhanced_clean.dog_stage == 'doggofloofer', 'dog_stage'] = 'doggo,floofer'

twitter_archive_enhanced_clean.loc[twitter_archive_enhanced_clean.dog_stage == '', 'dog_stage'] = 'None'
# -

# ###### Test: 

twitter_archive_enhanced_clean['dog_stage'].value_counts()

# Now, the data is completely untidy and have some `quality` issues. Lets proceed for further cleaning.
#
# ---
# ###### Issue: Remove unwanted rows and columns or the one that are not beneficial for us.

# ###### Define: 

# Removing unwanted rows and columns

# ###### Code: 

# +
twitter_archive_enhanced_clean = twitter_archive_enhanced_clean[twitter_archive_enhanced_clean.retweeted_status_id.isnull()]

# Drop Duplicate Tweet_IDs
twitter_archive_enhanced_clean.drop_duplicates(inplace=True)

twitter_archive_enhanced_clean.dropna(subset=['jpg_url'], inplace=True)

# Columns to be dropped
col = ['retweeted_status_id', 'retweeted_status_user_id', 'retweeted_status_timestamp', 'date_time', 'friends_count']
twitter_archive_enhanced_clean.drop(columns=col, inplace=True)

twitter_archive_enhanced_clean = twitter_archive_enhanced_clean.sort_values('dog_stage').drop_duplicates('tweet_id', keep='last')
# -

# ###### Test: 

twitter_archive_enhanced_clean.columns

# ###### Issue: Delete the image predictions columns

# ###### Define: 

# Delete the image predictions columns

# ###### Code: 

# +
# Append the first True predection to the list 'perdictions' and the level appended to list 'confidence_level',
# Otherwise, will append NaN.
predictions = []
confidence_level = []

def prediction_func(dataframe):
    if dataframe['p1_dog'] == True:
        predictions.append(dataframe['p1'])
        confidence_level.append(dataframe['p1_conf'])
    elif dataframe['p2_dog'] == True:
        predictions.append(dataframe['p2'])
        confidence_level.append(dataframe['p2_conf'])
    elif dataframe['p3_dog'] == True:
        predictions.append(dataframe['p3'])
        confidence_level.append(dataframe['p3_conf'])
    else:
        predictions.append('NaN')
        confidence_level.append(0)

twitter_archive_enhanced_clean.apply(prediction_func, axis=1)
twitter_archive_enhanced_clean['prediction'] = predictions
twitter_archive_enhanced_clean['confidence_level'] = confidence_level

col = ['img_num', 'p1', 'p1_conf', 'p1_dog', 'p2', 'p2_conf', 'p2_dog',\
       'p3', 'p3_conf', 'p3_dog', 'in_reply_to_status_id',\
       'in_reply_to_user_id', 'favourites_count']

# Deleteing unwanted columns
twitter_archive_enhanced_clean.drop(columns=col, inplace=True)
# -

# ###### Test: 

twitter_archive_enhanced_clean.info()

# ###### Issue: __`Twitter Archive Enhanced Data:`__ `Source` format is bad and can't be read easily.

# ###### Define: 

# Make `Source` format is good and can read easily.

# ###### Code: 

twitter_archive_enhanced_clean.source = twitter_archive_enhanced_clean.source.apply(lambda x: re.findall(r'>(.*)<', x)[0])

# ###### Test: 

twitter_archive_enhanced_clean.sample(5)

# ###### Issue: __`Twitter Archive Enhanced Data:`__ The columns `numerator` and `denominator` have invalid values.

# ###### Define: 

# The columns `numerator` and `denominator` have invalid values.

# ###### Code: 

# +
rating_temp = twitter_archive_enhanced_clean[twitter_archive_enhanced_clean.text.str.contains( r"(\d+\.?\d*\/\d+\.?\d*\D+\d+\.?\d*\/\d+\.?\d*)")].text

for i in rating_temp:
    x = twitter_archive_enhanced_clean.text == i
    twitter_archive_enhanced_clean.loc[x, 'rating_numerator'] = re.findall(r"\d+\.?\d*\/\d+\.?\d*\D+(\d+\.?\d*)\/\d+\.?\d*", i)
    twitter_archive_enhanced_clean.loc[x, 'rating_denominator'] = 10
# -

# ###### Test: 

twitter_archive_enhanced_clean[twitter_archive_enhanced_clean.text.isin(rating_temp)]

# ###### Issue: Cleaning decimal values in rating numerators.

# ###### Define: 

# Cleaning decimal values in rating numerators.

# ###### Code: 

# View tweets with decimals in rating in 'text' column
twitter_archive_enhanced_clean[twitter_archive_enhanced_clean.text.str.contains(r"(\d+\.\d*\/\d+)")]

ratings = twitter_archive_enhanced_clean.text.str.extract('((?:\d+\.)?\d+)\/(\d+)', expand=True)
ratings

twitter_archive_enhanced_clean.rating_numerator = ratings[0]

# ###### Test: 

twitter_archive_enhanced_clean[twitter_archive_enhanced_clean.text.str.contains(r"(\d+\.\d*\/\d+)")]

# ###### Issue: Convert the `null` values to `None` type

# ###### Define: 

# Convert the `null` values to `None` type

twitter_archive_enhanced_clean.info()

# ###### Code: 

twitter_archive_enhanced_clean.loc[twitter_archive_enhanced_clean['prediction'] == 'NaN', 'prediction'] = None
twitter_archive_enhanced_clean.loc[twitter_archive_enhanced_clean['rating_numerator'] == 'NaN', 'rating_numerator'] = 0

# ###### Test: 

twitter_archive_enhanced_clean.info()

# ###### Issue:  __`Twitter Archive Enhanced Data:`__ Replacing all lower case invalid names like (`a, an, etc.`) with `None`
#

# ###### Define: 

# Replacing all lower case invalid names like (`a, an, etc.`) with `None`

# ###### Code: 

twitter_archive_enhanced_clean.name = twitter_archive_enhanced_clean.name.str.replace('^[a-z]', 'None')

# ###### Test: 

twitter_archive_enhanced_clean.name.value_counts()

# ###### Issue:  __`Twitter Archive Enhanced Data:`__ Replacing all `O` invalid names with `None`

# ###### Define: 

# Replacing all `O` invalid names with `None`

# ###### Code: 

twitter_archive_enhanced_clean.name = twitter_archive_enhanced_clean.name.str.replace('O', 'None')

# ###### Test: 

twitter_archive_enhanced_clean[twitter_archive_enhanced_clean['name'].apply(len) == 1 ].name.value_counts()

# ###### Issue: Correcting Data types

# ###### Define: 

# Correcting Data types

# ###### Code: 

twitter_archive_enhanced_clean.dtypes

twitter_archive_enhanced_clean['tweet_id'] = twitter_archive_enhanced_clean['tweet_id'].astype(str)
twitter_archive_enhanced_clean['timestamp'] = pd.to_datetime(twitter_archive_enhanced_clean.timestamp)
twitter_archive_enhanced_clean['source'] = twitter_archive_enhanced_clean['source'].astype('category')
twitter_archive_enhanced_clean['rating_numerator'] = twitter_archive_enhanced_clean['rating_numerator'].astype(float)
twitter_archive_enhanced_clean['rating_denominator'] = twitter_archive_enhanced_clean['rating_denominator'].astype(float)
twitter_archive_enhanced_clean['favorite_count'] = twitter_archive_enhanced_clean['favorite_count'].astype(int)
twitter_archive_enhanced_clean['retweet_count'] = twitter_archive_enhanced_clean['retweet_count'].astype(int)
twitter_archive_enhanced_clean['followers_count'] = twitter_archive_enhanced_clean['followers_count'].astype(int)
twitter_archive_enhanced_clean['dog_stage'] = twitter_archive_enhanced_clean['dog_stage'].astype('category')

# ###### Test: 

twitter_archive_enhanced_clean.dtypes

# #### Storing Cleaned Data in `twitter_archive_master.csv` file  

twitter_archive_enhanced_clean.to_csv('twitter_archive_master.csv', index=False, encoding='utf-8')

# ###### The cleaned data has been successfully stored in `twitter_archive_master.csv` file
#
# ---

# ## Analyzing and Visualizing `twitter_archive_master` Data
#
# By gathering cleaned and analysis ready data from `twitter_archive_master.csv`.

master_df = pd.read_csv('twitter_archive_master.csv')
master_df.shape

master_df.info()

master_df.timestamp = pd.to_datetime(master_df.timestamp)
master_df.info()

master_df.head()

master_df.sample(5)

master_df.describe()

# ##### Number of Tweets Monthly

master_df['timestamp'].apply(lambda x: x.strftime('%Y-%m')).value_counts().sort_index()

selected_data = master_df['tweet_id'].groupby([master_df['timestamp'].dt.year, master_df['timestamp'].dt.month]).count()
selected_data.plot('line')
plt.title('Number of Tweets Monthly', size=15)
plt.xlabel('(Year, Month)')
plt.ylabel('Number of Tweets')
plt.savefig('images/Number-of-Tweets-Monthly.png', bbox_inches='tight');

# Most number of tweets posted were in December-2015 (350+ tweets). Afterwards the number of tweets decreased continuously until April-2016 and remained constant afterwards until the July-2017.

# ##### Distribution of Rating Numerator

ratings = master_df[(master_df.rating_numerator <= 14) & (master_df.rating_numerator.apply(float.is_integer))]
rating_counts = ratings.groupby(['rating_numerator']).count()['tweet_id']

plt.bar(np.arange(15), rating_counts)
plt.xticks(np.arange(15))
plt.xlabel('Rating Numerator')
plt.ylabel('Frequency')
plt.title('Distribution of Rating Numerators')
plt.savefig('images/Distribution-of-Rating-Numerator.png', bbox_inches='tight');

# Most rated tweets lie between `10-13`

# ##### Most used `Source`

master_df.source.value_counts()

sns.countplot(data=master_df, x='source')
plt.title('Most used Tweet Sources', size=15)
plt.xlabel('Tweets Sources')
plt.ylabel('Count')
plt.savefig('images/Most-used-Sources.png', bbox_inches='tight');

# The most popular source is `Twitter for iPhone` followed by the `Twitter Web Client` and `TweetDeck`

# ##### Correlation between Variables

sns.heatmap(master_df.corr(), cmap='Blues')
plt.title('Correlation Heatmap')
plt.savefig('images/Correlation-between-Variables.png', bbox_inches='tight');

# There are some weak and strong correlations.
# - `Favorite_count` and `Retweet_count` have a strong correlation
# - `Favorite_count` and `Followers_count` have a weak correlation

# ##### Most Frequent Dog Stages

# +
x = np.char.array(['Pupper', 'Doggo', 'Puppo', 'Doggo, Pupper', 'Floofer', 'Doggo, Puppo', 'Doggo, Floofer'])
y = np.array(list(master_df[master_df['dog_stage'] != 'None']['dog_stage'].value_counts())[0:7])
explode = (0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1) 
percent = 100.*y/y.sum()
patches, texts = plt.pie(y,  startangle=90, radius=1.8,  explode = explode)
labels = ['{0} - {1:1.2f} %'.format(i,j) for i,j in zip(x, percent)]

plt.legend(patches, labels,  bbox_to_anchor=(-0.1, 1.), fontsize=8);
plt.title('Most Frequent Dog Stages')
plt.axis('scaled')

plt.savefig('images/Most-Frequent-Dog-Stages.png', bbox_inches='tight');
# -

# `Pupper` is the most common dog found.

# ##### Relation b/w Retweets and Likes

# +
master_df.plot(kind='scatter', 
              x='favorite_count',
              y='retweet_count',
              alpha=0.5,
              figsize=(15, 11))
plt.xlabel('Number of Likes')
plt.ylabel('Number of Retweets')
plt.title('Relation b/w Retweets and Likes')

plt.savefig('images/Relation-between-Retweets-and-Likes.png', bbox_inches='tight');
# -

# - `Like` and `Retweets` have a strong correlation

StarcoderdataJupyterScriptsDedupFiltered