# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + cell_id="00000-583fceab-2062-436e-8e1d-c8b9fbe843fe" deepnote_cell_type="code" deepnote_to_be_reexecuted=false execution_millis=6 execution_start=1621522081250 source_hash="60c2cabd"
import re
import nltk
import pandas as pd
import numpy as np 
import wrangle
import unicodedata
from wrangle import wrangle_articles
import env
import warnings
warnings.filterwarnings("ignore")
# %matplotlib inline
import matplotlib.pyplot as plt
import seaborn as sns
#Wordclouds
from PIL import Image
from wordcloud import WordCloud
from wordcloud import ImageColorGenerator

# + cell_id="00035-23fff588-e25f-4567-9ebc-8c84bc7f3a38" deepnote_cell_type="code" tags=[]
articles_df = wrangle_articles()
# -

articles_df.head()

# +
ADDITIONAL_STOPWORDS = ['r', 'u', '2', 'ltgt', 'reuters']

def clean(text):
    'A simple function to cleanup text data'
    wnl = nltk.stem.WordNetLemmatizer()
    stopwords = nltk.corpus.stopwords.words('english') + ADDITIONAL_STOPWORDS
    text = (unicodedata.normalize('NFKD', text)
             .encode('ascii', 'ignore')
             .decode('utf-8', 'ignore')
             .lower())
    words = re.sub(r'[^\w\s]', '', text).split()
    return [wnl.lemmatize(word) for word in words if word not in stopwords]



# +
def show_counts_and_ratios(df, column):
    """
    This fucntion takes in a df and column name.
    Will produce a valuecounts for each label and the percetage of the data it represents
    """
    fof = pd.concat([df.is_fake.value_counts(),
                        df.is_fake.value_counts(normalize=True)], axis=1)
    fof.columns = ['n', 'percent']
    
    return fof

show_counts_and_ratios(articles_df, 'is_fake')
# -

df = articles_df
df.head()

df['text'] = df['text'].astype(str)
df['title'] = df['title'].astype(str)
df['clean_text'] = df['clean_text'].astype(str)
df['clean_title'] = df['clean_title'].astype(str)

df.head()

fake_words = clean(' '.join(df[df.is_fake == True].clean_text))
real_words = clean(' '.join(df[df.is_fake == False].clean_text))
all_words = clean(' '.join(df.clean_text))

# +
#word frequencies
fake_freq = pd.Series(fake_words).value_counts()
real_freq = pd.Series(real_words).value_counts()
all_freq = pd.Series(all_words).value_counts()

fake_freq.head(10)
# -

#top 10 fake words
fake_words[0:10]

#top 10 real words
real_words[0:10]

#top 20 of all words
all_words[0:20]

# +
word_counts = (pd.concat([all_freq, fake_freq, real_freq], axis=1, sort=True)
                .set_axis(['all', 'fake', 'real'], axis=1, inplace=False)
                .fillna(0)
                .apply(lambda s: s.astype(int)))

word_counts
# -

word_counts.sort_values(by='all', ascending=False).head(25)


word_counts.sort_values(by='all', ascending=False).tail(25)

# ratio 
word_counts_ratio  = (word_counts
 [(word_counts.fake > 10) & (word_counts.real > 10)]
 .assign(ratio=lambda df: df.fake / (df.real + .01))
 .sort_values(by='ratio')
 .pipe(lambda df: pd.concat([df.head(), df.tail()])))
word_counts_ratio

# +
#Barplot
(word_counts
 .assign(p_fake=word_counts.fake / word_counts['all'],
         p_real=word_counts.real / word_counts['all'])
 .sort_values(by='all')
 [['p_fake', 'p_real']]
 .tail(20)
 .sort_values('p_real')
 .plot.barh(stacked=True))

plt.title('Proportion of Fake News and Real News for the 20 most common words')

# +
#wordclouds
all_cloud = WordCloud(background_color='grey', height=1000, width=400, colormap="magma").generate(' '.join(all_words))
fake_cloud = WordCloud(background_color='grey', height=600, width=800, colormap="magma").generate(' '.join(fake_words))
real_cloud = WordCloud(background_color='grey', height=600, width=800, colormap="magma").generate(' '.join(real_words))

plt.figure(figsize=(10, 8))
axs = [plt.axes([0, 0, .5, 1]), plt.axes([.5, .5, .5, .5]), plt.axes([.5, 0, .5, .5])]

axs[0].imshow(all_cloud)
axs[1].imshow(fake_cloud)
axs[2].imshow(real_cloud)

axs[0].set_title('All Words')
axs[1].set_title('Fake')
axs[2].set_title('Real')

for ax in axs: ax.axis('off')

# +
top_20_real_bigrams = (pd.Series(nltk.ngrams(real_words, 2))
                      .value_counts()
                      .head(20))

top_20_real_bigrams.head()

# +
top_20_real_bigrams.sort_values().plot.barh(color='teal', width=.9, figsize=(10, 6))

plt.title('20 Most frequently occuring real bigrams')
plt.ylabel('Bigram')
plt.xlabel('# Occurances')

# make the labels pretty
ticks, _ = plt.yticks()
labels = top_20_real_bigrams.reset_index()['index'].apply(lambda t: t[0] + ' ' + t[1])
_ = plt.yticks(ticks, labels)

# +
top_20_fake_bigrams = (pd.Series(nltk.ngrams(fake_words, 2))
                      .value_counts()
                      .head(20))

top_20_fake_bigrams.head()

# +
top_20_fake_bigrams.sort_values().plot.barh(color='teal', width=.9, figsize=(10, 6))

plt.title('20 Most frequently occuring fake bigrams')
plt.ylabel('Bigram')
plt.xlabel('# Occurances')

# make the labels pretty
ticks, _ = plt.yticks()
labels = top_20_fake_bigrams.reset_index()['index'].apply(lambda t: t[0] + ' ' + t[1])
_ = plt.yticks(ticks, labels)
# -

#Trumpmask cloud wordcloud






StarcoderdataJupyterScriptsDedupFiltered