# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

subreddits = ['news', 'politics', 'worldnews', 'Liberal', 'progressive', 'democrats', 'Conservative', 'The_Donald', 'Republican']
news_subreddits = ['news', 'politics', 'worldnews']
left_subreddits = ['Liberal', 'progressive', 'democrats']
right_subreddits = ['Republican'] #'Conservative', 'The_Donald', 

import os
os.chdir('../../..')

import convokit
from convokit import Corpus, FightingWords

root_dir = "/Users/calebchiam/Documents/datasets/"

subreddit_corpora = dict()
for subreddit in left_subreddits + right_subreddits:
    subreddit_corpora[subreddit] = Corpus(filename=root_dir + subreddit + "-filtered-labelled-small")

for subreddit, corpus in subreddit_corpora.items():
    print(subreddit)
    corpus.print_summary_stats()
    print()

# +
political_corpus = None

for subreddit, corpus in subreddit_corpora.items():
    if political_corpus is None:
        political_corpus = corpus
    else:
        political_corpus = political_corpus.merge(corpus)
# -

political_corpus.print_summary_stats()

FW = FightingWords(l1_selector=lambda utt: utt.meta['subreddit'] == 'democrats', 
                  l2_selector=lambda utt: utt.meta['subreddit'] == 'Republican', ngram=(1,2))

# %matplotlib qt
FW.summarize(political_corpus)

FW.annot_method

FW.top_k

# %matplotlib inline
FW.plot_fighting_words(max_label_size=15)

FW.top_k = 15

FW.get_zscore('reform')

FW.get_zscore('control')

FW.get_zscore('liberals')

FW.get_zscore('idiots')

FW.get_zscore('trump')

FW.get_zscore('cuck')



StarcoderdataJupyterScriptsDedupFiltered