# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# <h1 style="text-align:center;color:tomato;">Wrangle and Analyze Data</h1>

# ## Importing the libraries

# !pip3 install statsmodels

# +
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt


import requests as req
import tweepy
import json
import re
import statsmodels.api as sm
import matplotlib.patches as mpatches

# %matplotlib inline
# -

# <h1 style="text-align:center;color:tomato;"> Gathering of Data</h1>

# #### Twitter Enhanced Dataset (Via local file)

df_enhanced=pd.read_csv('twitter-archive-enhanced.csv')

df_enhanced.head(3)

# # Image Predictions (Via url)

# +
url = "https://d17h27t6h515a5.cloudfront.net/topher/2017/August/599fd2ad_image-predictions/image-predictions.tsv"
response = requests.get(url)

with open('image-predictions.tsv', mode ='wb') as file:
    file.write(response.content)

#Read TSV file
df_image_pred = pd.read_csv('image-predictions.tsv', sep='\t' )

# -

df_img_pred.head(3)

# ### Twitter API ( Via API)

# +
consumer_key = ''
consumer_secret = ''
access_token = ''
access_secret = ''

auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)

api = tweepy.API(auth,wait_on_rate_limit=True)
# -

#Storing list of tweets ids to get their content later
tweets_id=df_enhanced["tweet_id"]

# +
#Creating a list to store tweet_ids which we fail to get content from (possibly deleted)
del_tweets=[]

#creating and opening file to write the data into
with open('tweet_json.txt',mode='w') as file:
    for uid in tweets_id:
        try:
            #fetching tweet content and storing it 
            tweet=api.get_status(uid,tweet_mode='extended')
            json.dump(tweet._json,file)
            #Writing new observation in a newline 
            file.write("\n")
        except:
            del_tweets.append(uid)
            print("Error fetching Tweet ID: ",uid)
# -

#To understand how our extracted JSON format is 
with open('tweet_json.txt','r') as data:
    tweet_json = data.readline()
    print(tweet_json)

# +
#Converting our data into DataFrame
df_tweet=pd.read_json('tweet_json.txt',orient='records',lines=True)


# -

df_tweet.head(3)

# #### Removing the unnecessary columns

df_tweet.columns

df_tweet=df_tweet[["id","favorite_count","retweet_count","retweeted"]]
df_tweet.head()

#
# <h1 style="text-align:center;color:tomato;">Assessing Data</h1>

# ### Enhanced Dataset

df_enhanced.head()

df_enhanced.info()

# * Many Columns the datatype is wrong (for eg. in_reply_to,timestamp,retweeted_status_id tec.)
# * Missing Values in dog stages columns (doggo,floofer,pupper,puppo) are registered as None
# * Some Exapnded URL's are missing
#

df_enhanced.query('doggo=="None" and floofer=="None" and pupper=="None" and puppo=="None"')

# * Stages of dogs have not been mentioned and been names as "None"

df_enhanced['source'].value_counts()

# * Sources are diificult to read and interpret

# ### Image Prediction Dataset

df_img_pred

df_img_pred.info()

df_img_pred.describe()

# * No missing values as such

len(df_img_pred.query('p1_dog==False and p2_dog==False and p3_dog==False'))

# * Some objects may not be dog at all as they have been failed to classified as dogs by the algorithm

# ### Twitter API

df_tweet

df_tweet.info()

df_tweet.describe()

# ## Summary 

#
# ### Quality
#
# #### df_enhanced
# * Data types are wrongly written in many places
# * Sources are not clearly understandable/readable
# * Retweets are present in dataset
# * Missing Values in dog stages columns (doggo,floofer,pupper,puppo) are registered as None
# * Change all numerator to 10 and scale numerator accordingly for better comparison
#
# #### df_image_pred
# * Some images arent even dog by the algorithm but still classified as some
# * Coverting the breed into categorical data
#
# ### Tideness
#
# * Merging all the datasets into one
# * Indiviual dogs stages columns need to be converted into a single categorical column
# * Unnecessary columns need to be removed
#

# <h1 style="text-align:center;color:tomato;">Data Cleaning</h1>

# * First we will make make copies of the dataset

df_enhanced_clean=df_enhanced.copy()
df_img_pred_clean=df_img_pred.copy()
df_tweet_clean=df_tweet.copy()

# ## Quality
#
# ### 1.Twitter enhanced dataset
#

# ### 1.1 
# #### Define
# * Correcting the datatype in columns -timestamp,retweeted_status_timestamp and converting various id's to string datatype

# #### Code
# * Will use pd.to_datetime to convert into datatype and astype(str) to convert into string

# To datetime
df_enhanced_clean.timestamp = pd.to_datetime(df_enhanced_clean.timestamp)
df_enhanced_clean.retweeted_status_timestamp = pd.to_datetime(df_enhanced_clean.retweeted_status_timestamp)
# To string
df_enhanced_clean.in_reply_to_status_id=df_enhanced_clean.in_reply_to_status_id.astype(str)
df_enhanced_clean.in_reply_to_user_id=df_enhanced_clean.in_reply_to_user_id.astype(str)
df_enhanced_clean.retweeted_status_id=df_enhanced_clean.retweeted_status_id.astype(str)
df_enhanced_clean.retweeted_status_user_id=df_enhanced_clean.retweeted_status_user_id.astype(str)

# #### Test

df_enhanced_clean.info()

# ### 1.2

# #### Define 
# * Removing the retweeted tweets. they are present in dataset and needs to be removed as they account for duplicate value 

# #### Code 
# * Remove the tweets which have no null status in retweet_status_id

df_enhanced_clean=df_enhanced_clean.query('retweeted_status_id == "nan"')

# #### Test

len(df_enhanced_clean.query('retweeted_status_id != "nan"'))

# ### 1.3 
#

# #### Define
# * Missing values in dog are shown as none

# #### Code
# * Change the none into NAN 

df_enhanced_clean.floofer=df_enhanced_clean.floofer.replace("None",np.nan,regex=True)
df_enhanced_clean.pupper=df_enhanced_clean.pupper.replace("None",np.nan,regex=True)
df_enhanced_clean.doggo=df_enhanced_clean.doggo.replace("None",np.nan,regex=True)
df_enhanced_clean.puppo=df_enhanced_clean.puppo.replace("None",np.nan,regex=True)

# #### Test

df_enhanced_clean.info()


# ### 1.4
#
# #### Define 
#
# * Types of sources not human readable/clearly understood, since they are written in HTML tags format, it is difficult for a normal person to understand.
#
# ####  Code
#
# * Extract the text between the anchor tags to extract the actual source used

# +
def source(x):
    source_list=[]
    pattern = re.compile('llow">(.*?)</a>')
    for tag in x['source']:
        m = re.search(pattern, tag)
        src=m.group(1)
        source_list.append(src)
    return source_list

src=source(df_enhanced_clean)
df_enhanced_clean['source']=src
# -

# #### Test

df_enhanced_clean['source'].value_counts()


# ### 1.5
#
# #### Define 
#
# * Some of the expanded URLs are missing. I found out from the Twitter API Documentation,tweet_id is an extention to 'https://twitter.com/dog_rates/status/' which gives us the URL to the tweet we are looking at
#
# #### Code 
#
# * For places where expanded URL is null, we will add the https://twitter.com/dog_rates/status/ and tweet_id column, and convert into a string to store in.(We will be using pandas apply function)

# +
def form_link(x):
    if pd.notnull(x['expanded_urls']):
        return x
    else:
        tid = x['tweet_id']
        x['expanded_urls'] = 'https://twitter.com/dog_rates/status/' +str(tid)
        return x


df_enhanced_clean = df_enhanced_clean.apply(form_link, axis=1)
# -

# #### Test

df_enhanced_clean[df_enhanced_clean['expanded_urls'].isnull()]

# ### 1.6
# #### Define 
#
# * We can bring the denominator to 10 and asjust the numberator accordingly
#
# #### Code
#
# * We will divide the numerator column by the denominator column and then multiply it by 10 to get the rating out of ten, and then drop the denominator column which will no longer be needed

# +
df_enhanced_clean.rating_numerator=(df_enhanced_clean.rating_numerator/df_enhanced_clean.rating_denominator)*10


# -

df_enhanced_clean[df_enhanced_clean['rating_denominator']!=10]

df_enhanced_clean.loc[2335,'rating_numerator']=9
df_enhanced_clean.loc[313,'rating_numerator']=13

df_enhanced_clean.drop(['rating_denominator'],axis=1,inplace=True)

# #### Test

df_enhanced_clean.info()

# ### 2. Image Prediction Dataset
#
# ### 2.1
#
# #### Define 
#
# There are some cases when the algorithm has predicted that the given image is not a dog
#
# #### Code
#
# Filter out the instances where the algortihm predicts that the image is not of a dog in all 3 instances

df_img_pred_clean=df_img_pred_clean.query('p1_dog == True or p2_dog == True or p3_dog == True')

# #### Test

len(df_img_pred_clean.query('p1_dog == False and p2_dog == False and p3_dog == False'))

# ### 2.2
#
# #### Define 
#
# Dogs Breeds are mentioned as strings but they are categorical data
#
# #### Code
#
# Convert datatype into category

df_img_pred_clean['p1']=df_img_pred_clean['p1'].astype('category')
df_img_pred_clean['p2']=df_img_pred_clean['p2'].astype('category')
df_img_pred_clean['p3']=df_img_pred_clean['p3'].astype('category')

# #### Test

df_img_pred_clean.info()

# ### 2.3 
#
# #### Define 
#
# Breed names are not capatalized and there are underscore between some breed names.
#
# #### Code 
#
# Capitalising every string under columns p1,p2,p3 using pandas  .capitalize() function and also removing the underscore with python str.replace function

#Capitalizing Breed Names
df_img_pred_clean['p1'] = df_img_pred_clean.p1.str.capitalize()
df_img_pred_clean['p2'] = df_img_pred_clean.p2.str.capitalize()
df_img_pred_clean['p3'] = df_img_pred_clean.p3.str.capitalize()

#Replacing _ with ' ' in the columns
df_img_pred_clean['p1']=df_img_pred_clean['p1'].replace("_",r' ',regex=True)
df_img_pred_clean['p2']=df_img_pred_clean['p2'].replace("_",r' ',regex=True)
df_img_pred_clean['p3']=df_img_pred_clean['p3'].replace("_",r' ',regex=True)

# #### Test

df_img_pred_clean.head()

# ## Tidiness
#
# ### Twitter Enhaced Dataset
#
# ### 1.1
#
# #### Define
# Removing some non useful columns  
#
#
#
#

# #### Code

df_enhanced_clean.columns

# +

df_enhanced_clean.drop(['retweeted_status_timestamp','retweeted_status_user_id','retweeted_status_id'],axis=1,inplace=True)

# -

# #### Test

df_enhanced_clean.columns

# ### 1.2 
#
# <b> Define </b>
#
# The different stages of dogs are expanded as different variables in the dataset whereas in reality its a should be a single categorical value denoting the stage of a dog
#
# <b> Code </b>
#
# While keeping other columns constant, the columns ('doggo' ,'floofer', 'pupper', 'puppo') will be melted into one column
#
# We have to keep in mind that one dog can be in more than one stage since the terms are vaguely defined`

# +
# Create new column for dog stages 
df_enhanced_clean['dog_stage'] = 'None'

# Function that will be applied to each row (changes dog_stage value)
def dog_stage(row):
    stage = []
    if row['doggo'] == 'doggo':
        stage.append('doggo')
    if row['floofer'] == 'floofer':
        stage.append('floofer')
    if row['pupper'] == 'pupper':
        stage.append('pupper')
    if row['puppo'] == 'puppo':
        stage.append('puppo')
    if len(stage)==0: # Default to 'None' if list is empty
        row['dog_stage'] = 'None'
    else: 
        # If there is more than 1 stage
        row['dog_stage'] = ','.join(stage)
    
    return row

df_enhanced_clean = df_enhanced_clean.apply(dog_stage, axis=1)


# -

df_enhanced_clean.drop(['doggo','floofer','pupper','puppo'],axis=1,inplace=True)

# #### Test

df_enhanced_clean.dog_stage.value_counts()

# ### 2.0
#
# #### Define 
#
# Creating a Master Dataset
#
# #### Code 
#
# Merging all tables using tweet_id as the primary key and by inner join, so we have a tight dataset of tweets for which we have all data available

df_tweet_clean.head()

df_enhanced_clean.head()

df_master_1=pd.merge(left=df_enhanced_clean,right=df_tweet_clean,left_on='tweet_id',right_on='id',how='inner')

df_master_1.drop(['id','retweeted'],axis=1,inplace=True)
df_master_1.head()

df_master=pd.merge(left=df_master_1,right=df_img_pred_clean,on='tweet_id',how='inner')

# #### Test

df_master.head(3)

df_master.shape

# +
#Changing source into category
df_master['source']=df_master['source'].astype('category')
df_master['dog_stage']=df_master['dog_stage'].astype('category')

df_master.info()
# -

# ## Storing the dataset externally

df_master.to_csv('twitter_archive_master.csv',index=False)

df=pd.read_csv('twitter_archive_master.csv')
df.head(3)

# <h1 style="color:tomato;text-align:center">Analysis</h1>

# ## Q1) Breed of Dog which recieve more retweets?

# +
df_1 = df_master.groupby('p1')['retweet_count'].sum().reset_index()
df_sorted = df_favorite.sort_values('retweet_count', ascending=False).head(15)
retweet = df_sorted['retweet_count']
breed = df_sorted['p1']

fig, ax = plt.subplots(figsize=(10,7))
fav = plt.barh(breed, retweet)

plt.ylabel('Breed of Dog')
plt.xlabel('Retweet count')
plt.title('Retweets per breed');
# -

# > * Golden Retriver has the most number of retweets as evident from the above plot
# * The retweets range from 100k to 500k

# ## Q2)What are number of favorites by the breed

df_favorite = df.groupby('p1')['favorite_count'].sum().reset_index()
df_sorted = df_favorite.sort_values('favorite_count', ascending=False).head(10)
ser_fav = df_sorted['favorite_count']
ser_breed = df_sorted['p1']
#E90552
fig, ax = plt.subplots(figsize=(8,6))
fav = plt.barh(ser_breed, ser_fav)
plt.ylabel('Dog Breed')
plt.xlabel('Number of Favorites')
plt.title('Number of Favorites by Breed')

#
# >* As we can see from the graph Golden Retriever is the most favortite dog 
# * Every breed listed here  had atleast over 200K favorites and peaked at over 1.6 Million
#

# ## Q3)Is there a correlation between favorites and retweets?

# !pip3 install seaborn

import seaborn as sb

# +

# Plot scatterplot of retweet vs favorite count
sb.lmplot(x="retweet_count", 
           y="favorite_count", 
           data=df_master,
           size = 5,
           aspect=1.3,
           scatter_kws={'alpha':1/5})
plt.title('Favorite vs. Retweet Count')
plt.xlabel('Retweet Count')
plt.ylabel('Favorite Count');
# -

# > As we can see there is a high correlation between favorites and retweet count

# ## Q4) What are most popular dog names?

df_master.name.value_counts()[2:7].plot('barh', figsize=(15,8))
plt.title='Most Common Dog Names'
plt.xlabel("Number of Dogs");

# > We found out that Cooper is the most widely used dog name out of the name data that was available

# ## Q5) What are Proportion of various dog stages?

# +
df_stages = df[df['dog_stage'] != "None"]
fig, ax = plt.subplots(figsize=(10,10))

df_stages['dog_stage'].value_counts().plot(kind = 'pie', ax = ax, label = 'Dog Stage', autopct='%1.1f%%',explode=[0,0,0,0,0,0.125,0.125])
plt.legend();
# -

# >I first removed the values and then  created a pie chart to see what was the distribution of dog_stages among the WeRateDogs' Dog population
#
# >* More than half of it was covered by puppers: These are the dogs which are still in training hence a bit immature
# * 1/5th of the population was covered by the more mature Doggo population
#







StarcoderdataJupyterScriptsDedupFiltered