# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Apply Emotion Tracker
# Nov 14th 2019
#
# Testing that the emotion tracker works for utterances in corpus.

# +
# import required modules and set up environment
import os

# replace file path below with your own local convokit
os.chdir('/Users/marianneaubin/Documents/Classes/CS6742/cs6742-fork')

import convokit

from convokit import Corpus, Parser, EmoTracker, Transformer
import nltk
# -

# load corpus
corpus = convokit.Corpus(filename='../datasets/democrats-filtered-labelled-small')

corpus.print_summary_stats()

et = EmoTracker();
corpus = et.transform(corpus)

categories = ["emotional", "sadness", "negative_emotion", "shame", 
              "violence", "rage", "pain", "anger", "disgust", "hate", 
                "love", "politics"]

counter = 1
emotional_dist_dict = {"emotional": 0,
                       "sadness": 0, 
                       "negative_emotion": 0,
                       "shame": 0,
                       "violence":0,
                       "rage":0,
                       "pain":0,
                       "anger":0,
                       "disgust":0,
                       "hate":0, 
                       "love":0,
                       "politics":0}
for conv_id in corpus.conversations:
    conv = corpus.get_conversation(conv_id)
    for utt in conv.iter_utterances():
        if utt.meta["analysis"] != None:
            for cat in categories:
                if (utt.meta["analysis"][cat] != 0.0):
                    emotional_dist_dict[cat] = emotional_dist_dict[cat] + 1
            counter = counter + 1
print(emotional_dist_dict)

utter_ids = corpus.get_utterance_ids()

utter_ids[7000]

# +
pol_word_counter = 0
total_word_counter = 0
counter = 0
for x in range(0,3237456):
    utt = corpus.get_utterance(utter_ids[x])
    pol_word_counter = pol_word_counter + utt.meta['num_pol_refs']
    total_word_counter = total_word_counter + utt.meta['num_pol_refs_incidence']
    counter = counter + 1
    if counter % 100000 == 0:
        print(counter, " completed")
    
print("total # of words:", pol_word_counter)
print(total_word_counter)

# +
counter = 0
pol_words = {}
for x in range(0, 3237456):
    utt = corpus.get_utterance(utter_ids[x])
    utt_pol_words = utt.meta['pol_words']
    for y in utt_pol_words:
        if (y not in pol_words.keys()):
            pol_words[y] = 1
        else:
            pol_words[y] = pol_words[y] + 1
    counter = counter + 1
    if counter % 100000 == 0:
        print(counter, " completed")
        
print(pol_words)

# +
freqs = []
large_pol_words = []
for x in pol_words.keys():
    if pol_words[x] > 5000:
        freqs.append(pol_words[x])
        large_pol_words.append(x)
        
freqs, large_pol_words = (list(x) for x in zip(*sorted(zip(freqs, large_pol_words))))
freqs.reverse()
large_pol_words.reverse()
print(freqs)

# +
import matplotlib.pyplot as plt
import numpy as np
import matplotlib.pyplot as plt

objects = large_pol_words
y_pos = np.arange(len(objects))

plt.bar(y_pos, freqs, align='center', alpha=0.5)
plt.xticks(y_pos, objects, rotation='vertical')
plt.ylabel('Frequency')
plt.title('Most commonly used political words in Corpus')

plt.show()

## ISSUE: right now, this only works for single words. 
## will need to change the transformer so that it can also
## compute multiple words e.g. "bill of rights"

# +
## read in csv of relevant utts for sandy hook

import csv
#with open('../sandyhook_utterance_ids.csv', 'r') as f:
with open('../auroratheater_utterance_ids.csv', 'r') as f:
    reader = csv.reader(f)
    sh_list = list(reader)


# +
## recompute political freqs

pol_words_sh = {}
for x in sh_list:
    utt = corpus.get_utterance(x[0])
    utt_pol_words = utt.meta['pol_words']
    for y in utt_pol_words:
        if (y not in pol_words_sh.keys()):
            pol_words_sh[y] = 1
        else:
            pol_words_sh[y] = pol_words_sh[y] + 1
        
print(pol_words_sh)

# +
freqs_sh = []
large_pol_words_sh = []
for x in pol_words_sh.keys():
    freqs_sh.append(pol_words_sh[x])
    large_pol_words_sh.append(x)
        
freqs_sh, large_pol_words_sh = (list(x) for x in zip(*sorted(zip(freqs_sh, large_pol_words_sh))))
freqs_sh.reverse()
large_pol_words_sh.reverse()
print(freqs_sh)

# +
objects = large_pol_words_sh
y_pos = np.arange(len(objects))

plt.figure(figsize=(10,4))
plt.bar(y_pos, freqs_sh, align='center', alpha=0.5)
plt.xticks(y_pos, objects, rotation='vertical')
plt.ylabel('Frequency')
plt.title('Most commonly used political words in Sandy Hook Corpus')

plt.show()
# -

print(corpus.get_utterance(utter_ids[60]))

# ## Sandy Hook analysis
# Now we try to establish a time series of how many words there are per day after December 14, 2012 (Sandy Hook shooting day). Timestamp: 1355461200

# +
from datetime import datetime
import pandas as pd 

ps = 7

#Sandy hook dates
#start_date = '2012-12-14'
#end_date = '2012-12-22'
start_date = '2012-07-20'
end_date = '2012-07-28'

num_posts_sh = [0] * ps
times = pd.date_range(start=start_date,end=end_date,periods=ps+1)
times = np.array(times)
bin_times = times[:-1]

# convert to datetime object
times_temp = []
for i,x in enumerate(times):
    times_temp.append(pd.to_datetime(x))
times = times_temp    

for i,x in enumerate(sh_list):
    utt = corpus.get_utterance(x[0])
    posted_time = datetime.fromtimestamp(utt.timestamp)
    y = 0
    while (posted_time > times[y]):
        y = y + 1
    ## this gives us the timeframe to mark it as
    num_posts_sh[y-1] = num_posts_sh[y-1] + 1
print(num_posts_sh)
# -

plt.plot(bin_times,num_posts_sh)
plt.xticks(rotation='vertical')
plt.ylabel('Number of posts')
plt.title('Number of posts per day 7 days after Sandy Hook')
plt.show()

# +
from nltk.tokenize import word_tokenize

## function takes in utterance and counts total words
def count_total_words(utt):
    if utt.text != None:
        tokenized = word_tokenize(utt.text.lower())
        return len(tokenized)


# -

##check above function works as expected
print(count_total_words(corpus.get_utterance(utter_ids[400080])))
print(corpus.get_utterance(utter_ids[400080]))

# +
## next, I want the incidence rate per day of political words

inc_rate_sh = [0]*ps
total_pol_words_sh = [0]*ps
total_words_sh = [0]*ps

for i,x in enumerate(sh_list):
    utt = corpus.get_utterance(x[0])
    posted_time = datetime.fromtimestamp(utt.timestamp)
    y = 0
    while (posted_time > times[y]):
        
        y = y + 1
               
    ## this gives us the timeframe to mark it as
    total_pol_words_sh[y-1] = total_pol_words_sh[y-1] + utt.meta['num_pol_refs']
    total_words_sh[y-1] = total_words_sh[y-1] + count_total_words(utt)
    if y == 0:
        print(count_total_words(utt))

print(total_pol_words_sh)
print(total_words_sh)

for i in range(0, ps):
    if total_words_sh[i] != 0:
        inc_rate_sh[i] = total_pol_words_sh[i]/total_words_sh[i]
    
print(inc_rate_sh)
# -

plt.plot(bin_times,inc_rate_sh)
plt.xticks(rotation='vertical')
plt.ylabel('Political words/total words')
plt.title('Incidence rate of political words in utterances')
plt.show()

StarcoderdataJupyterScriptsDedupFiltered