# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .r
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: R
#     language: R
#     name: ir
# ---

# # DIC LAB 1 Problem 2 : Study response to an event

# Define the limit of numbers to be searched

LIMIT = 20000

# Declare all the search, strings which will be searched in a vector, following are the hastags for SuperBowl

superBowlStrings = c("#superbowl","#Patriots","#Falcons","#SB51")

# Declare your custom search strings, which will be searched, in a vector. Leave it empty vector if no custom string.

# Eg : Empty Vector will look like customUserSearchStrings = c("")

# **NOTE : Make sure you empty leave customUserSearchStrings vector empty if you want to see the results from superBowlStrings**

customUserSearchStrings = c("") #c("#NASA")

# Create a boolean variable to check if the custom search string is present or not. **Custom search will be given preference over SuperBowl search.** Number of search limit will change too i.e. **LIMIT = 1000**

isCustomSearch = FALSE
if(customUserSearchStrings[1]!=""){
    isCustomSearch = TRUE
    LIMIT = 1000
    userSearchStrings = customUserSearchStrings
}else{
    isCustomSearch = FALSE
    LIMIT = 20000
    userSearchStrings = superBowlStrings
}

# Define all the libraries which needs to be set up for operations below

library("twitteR")
library("DBI")
library("RSQLite")
library("gtools")
library("bitops")
library("ggplot2")
library("RCurl")
library("RJSONIO")
library("ggmap")
library("sp")
library("mapdata")
library("maptools")
library("scales")
library("maps")
Sys.setlocale(category = "LC_ALL", locale = "C")

# Setup the twitter app key for authentication

setup_twitter_oauth('YOUR KEY')

# Following function fetches the state and country from the address_components field in json response .Following is the response

# "address_components" : [ { "long_name" : "Buffalo", "short_name" : "Buffalo", "types" : [ "locality", "political" ] }, { "long_name" : "<NAME>", "short_name" : "<NAME>", "types" : [ "administrative_area_level_2", "political" ] }, { "long_name" : "New York", "short_name" : "NY", "types" : [ "administrative_area_level_1", "political" ] }, { "long_name" : "United States", "short_name" : "US", "types" : [ "country", "political" ] } ],

# It will generate the state as New York and country as United States.

parseAddressComponents <- function(addr_comp){
    result = tryCatch(
        {
            addrSize = length(addr_comp)
            stateName = ""
            countryName = ""
            for(i in 1:addrSize){
                if(addr_comp[[i]]$types[[1]] == "administrative_area_level_1"){
                    stateName = addr_comp[[i]]$long_name
                }
                if(addr_comp[[i]]$types[[1]] == "country"){
                    countryName = addr_comp[[i]]$long_name
                }
            }
            if(stateName == ""){
                stateName = NA
            }
            if(countryName == ""){
                countryName = NA
            }
            return (c(stateName,countryName))
        },
        error=function(cond) {
            return (c(NA,NA))
        },
        warning=function(cond) {
        },
        finally={
        }
    )
    return (result);
}

# This function generates the url which will be used to query the Google server. You have to pass the address for this function

# Eg : https://maps.google.com/maps/api/geocode/json?address=Buffalo&key=YOUR_KEY_GOOGLE&sensor=false

getGeoCodeUrl <- function(address, return.call = "json", sensor = "false") {
    root <- "https://maps.google.com/maps/api/geocode/"
    url <- paste(root, return.call, "?address=", address,"&key=YOUR_KEY_GOOGLE","&sensor=", sensor, sep = "")
    return(URLencode(url))
}

# This function generates the url which will be used to query the Google server. You have to pass the latitude and longitude for this function

# Eg. https://maps.google.com/maps/api/geocode/json?address=42.88644679999999,-78.8783689&key=AIzaSyCskJdqaLuNJBJ58IHDqfsjJAm9jkZbcKM&sensor=false

getLatLngURL <- function(lat, lng, return.call = "json", sensor = "false") {
    root <- "https://maps.google.com/maps/api/geocode/"
    latlng = paste(lat, lng, sep = ",")
    url <- paste(root, return.call, "?address=", latlng,"&key=<KEY>","&sensor=", sensor, sep = "")
    return(URLencode(url))
}

# The below code decomposes the json response from the Google Map API code

# The output response is as follows for location Buffalo

# { "results" : [ { "address_components" : [ { "long_name" : "Buffalo", "short_name" : "Buffalo", "types" : [ "locality", "political" ] }, { "long_name" : "Erie County", "short_name" : "Erie County", "types" : [ "administrative_area_level_2", "political" ] }, { "long_name" : "New York", "short_name" : "NY", "types" : [ "administrative_area_level_1", "political" ] }, { "long_name" : "United States", "short_name" : "US", "types" : [ "country", "political" ] } ], "formatted_address" : "Buffalo, NY, USA", "geometry" : { "bounds" : { "northeast" : { "lat" : 42.9664549, "lng" : -78.795157 }, "southwest" : { "lat" : 42.826023, "lng" : -78.9337276 } }, "location" : { "lat" : 42.88644679999999, "lng" : -78.8783689 }, "location_type" : "APPROXIMATE", "viewport" : { "northeast" : { "lat" : 42.9664549, "lng" : -78.795157 }, "southwest" : { "lat" : 42.826023, "lng" : -78.9142665 } } }, "place_id" : "ChIJoeXfUmES04kRcYEfGKUEI5g", "types" : [ "locality", "political" ] } ], "status" : "OK" }

# From the above json repsonse the data is extracted in a data frame shown in following codes

# We remove all those locations which are not in United State or which are in United State but no information about the state.

getGeoCode <- function(address,userScreenName,verbose=FALSE) {
    if(verbose) cat(address,"\n")
    url <- getGeoCodeUrl(address)
    doc <- getURL(url)
    docData <- fromJSON(doc,simplify = FALSE)
    
    if(docData$status=="OK") {
        
        lat <- docData$results[[1]]$geometry$location$lat
        lng <- docData$results[[1]]$geometry$location$lng        
        
        address_component <- parseAddressComponents(docData$results[[1]]$address_components)
        state = address_component[1]
        country = address_component[2]
        
        if(!invalid(country) & country == "United States"){
            
            if(invalid(state)){
                latlongUrl = getLatLngURL(lat,lng)
                latlngDoc <- getURL(latlongUrl)
                latlngDocData <- fromJSON(latlngDoc,simplify = FALSE)
    
                if(latlngDocData$status=="OK") {
                    address_component_latlng <- parseAddressComponents(latlngDocData$results[[1]]$address_components)
                    latlngState = address_component_latlng[1]
                    latlngCountry = address_component_latlng[2]
                    return(c(userScreenName, lat, lng, latlngState, latlngCountry))
                }
            }else{
                return(c(userScreenName, lat, lng, state, country))
            }
        } 
        Sys.sleep(0.5)
    }
}

# 1st line creates a sqlite db file

# 2nd line store and load tweets database which is backend registered

# 3rd line store the tweets in a table named "tweets", which is automatically provided by twitterR

if(!isCustomSearch){
    mydbName = toString("DIClab1.db")
}else{
    mydbName = toString("custom.db")
}
conn = dbConnect(SQLite(), dbname = mydbName)
register_sqlite_backend(mydbName)

# Searches the tweets from the user input search string vector and equally divides them in equal chunks per input string in vector. So that the total sum of all the tweets is LIMIT, i.e 20000 here.

size = length(userSearchStrings)
for(searchStr in userSearchStrings){
    tweets <- searchTwitter(searchStr,LIMIT/size)
    store_tweets_db(tweets)
}

# Makes a connection with the database

con = dbConnect(SQLite(), dbname = mydbName)

# Query the database for all the screen names of the users.

tweetsUserNames = dbGetQuery(con,"SELECT screenName from tweets")

# Generate a vector by taking transpose of the above result.

userNamesVec = c(t(tweetsUserNames))

# Prints a few user's screen names

head(userNamesVec)

# Queries the twitter with all the screen names to get the user information

lookupUserNames <- lookupUsers(userNamesVec)

# Converts the data collected in the previous step to a data frame

userNamesDf = twListToDF(lookupUserNames)

# Save this data frame so that the previous heavy work needs not to be repeated everytime

if(!isCustomSearch){
    saveRDS(userNamesDf,file = "userNamesDf.Rda")
}else{
    saveRDS(userNamesDf,file = "customUserNamesDF.Rda")
}

# Load the userNameDf from the saved file

if(!isCustomSearch){
    userNamesDf = readRDS(file = "userNamesDf.Rda")
}else{
    userNamesDf = readRDS(file = "customUserNamesDF.Rda")
}

# Print a few items of the data frame

head(userNamesDf)

# This is the heart of the whole code. It generates a data frame which will contain the screen name, latitude, longitude, state and country of the all the users. We keep storing the data frames as they arrive so that in case of any problem we can resume from the failure point.

# NOTE: Here in this experiment we generated some 5000+ locations.

# +
userLocationDataDf = data.frame(screenName = as.character(),Lat = double(),Long = double(),State = as.character(), Country = as.character())

sizeOfDf = length(userNamesDf$screenName)

for(i in 1:sizeOfDf){
    tempUserLocation = userNamesDf$location[i]
    tempUserScreenName = userNamesDf$screenName[i]
    if(tempUserLocation != ""){
        tempVec = getGeoCode(tempUserLocation,tempUserScreenName)
        if(!invalid(tempVec)){
            tempDf = data.frame(screenName = as.character(tempVec[1]),Lat = as.numeric(tempVec[2]),Long = as.numeric(tempVec[3]),State = as.character(tempVec[4]),Country = as.character(tempVec[5]))
            userLocationDataDf = rbind(userLocationDataDf,tempDf)
            if(!isCustomSearch){
                saveRDS(userLocationDataDf,file = "userLocationDataDf.Rda")
            }else{
                saveRDS(userLocationDataDf,file = "customUserLocationDataDf.Rda")
            }
        }
    }
}
# -

# Save the above generated data frame since the above process is time consuming.

if(!isCustomSearch){
    saveRDS(userLocationDataDf,file = "userLocationDataDf.Rda")
}else{
    saveRDS(userLocationDataDf,file = "customUserLocationDataDf.Rda")
}

# Load the data frame for further queries

if(!isCustomSearch){
    userLocationDataDf = readRDS(file = "userLocationDataDf.Rda")
}else{
    userLocationDataDf = readRDS(file = "customUserLocationDataDf.Rda")
}

# Print and check if the data frame generated is in right format or not

head(userLocationDataDf)

# Prints the number of items in the data frame

print(length(userLocationDataDf$screenName))

# This code basically aggregates the data as per the different state and club them together. This code also uses mean function to generate the latitude and logitude of the state the mean value of them will still be in the state.  

aggDataDf = aggregate(x = list(Latitude = userLocationDataDf$Lat, Longitude = userLocationDataDf$Long), by = list(State=userLocationDataDf$State), FUN = mean)

# This part of code generates the frequency table from the userLocationDataDf data frame per state.

tweetLocationFreqDf = aggregate(x = list(Tweet_Count = userLocationDataDf$State), list(userLocationDataDf$State), length)

# Merges the two data frames generated above

aggUserLocationDataDf = data.frame(aggDataDf,tweetLocationFreqDf)

# Save this data frame before moving further

if(!isCustomSearch){
    saveRDS(aggUserLocationDataDf,file = "aggUserLocationDataDf.Rda")
}else{
    saveRDS(aggUserLocationDataDf,file = "aggCustomUserLocationDataDf.Rda")
}

# Reload the data in a data frame again for futher processing

if(!isCustomSearch){
    aggUserLocationDataDf = readRDS(file = "aggUserLocationDataDf.Rda")
}else{
    aggUserLocationDataDf = readRDS(file = "aggCustomUserLocationDataDf.Rda")
}

# Print and check if the data frame is in right format or not.

head(aggUserLocationDataDf)

# get_map give the map of the specified location.

USAMap = get_map(location = 'United States', zoom = 4)

# This code generates the points as per the data provided. We need to pass the map, generated in the previous step and all the points which needs to be plotted

mapPoints <- ggmap(USAMap) +
    geom_point(aes(x = aggUserLocationDataDf$Longitude, y = aggUserLocationDataDf$Latitude, size = Tweet_Count), data = aggUserLocationDataDf, color = "blue")

# Draws the map with the marked points. 

mapPoints

# # References 

# 1. https://cran.r-project.org/web/packages/twitteR/twitteR.pdf
# 2. http://geoffjentry.hexdump.org/twitteR.pdf
# 3. https://developers.google.com/maps/documentation/geocoding/start#sample-request
# 4. https://gist.github.com/josecarlosgonz/6417633
# 5. https://cran.r-project.org/web/packages/RCurl/RCurl.pdf
# 6. http://geoffjentry.hexdump.org/twitteR.pdf
# 7. https://cran.r-project.org/web/packages/twitteR/twitteR.pdf
# 8. https://cran.r-project.org/web/packages/ggmap/ggmap.pdf



StarcoderdataJupyterScriptsDedupFiltered