# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3.8.8 64-bit (conda)
#     name: python3
# ---

# + [markdown] papermill={"duration": 0.028289, "end_time": "2021-09-16T16:18:48.494895", "exception": false, "start_time": "2021-09-16T16:18:48.466606", "status": "completed"} tags=[]
# # EDA on German Fake News Dataset

# + _cell_guid="b1076dfc-b9ad-4769-8c92-a6c4dae69d19" _uuid="8f2839f25d086af736a60e9eeb907d3b93b6e0e5" papermill={"duration": 1.642953, "end_time": "2021-09-16T16:18:50.164299", "exception": false, "start_time": "2021-09-16T16:18:48.521346", "status": "completed"} tags=[]
import os
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
import nltk

#from nltk.corpus import stopwords
from dotenv import find_dotenv, load_dotenv
load_dotenv(find_dotenv())


pd.set_option('display.max_rows', 100)

# + [markdown] papermill={"duration": 0.026218, "end_time": "2021-09-16T16:18:50.217641", "exception": false, "start_time": "2021-09-16T16:18:50.191423", "status": "completed"} tags=[]
# ## Read data

# + papermill={"duration": 4.974594, "end_time": "2021-09-16T16:18:55.218688", "exception": false, "start_time": "2021-09-16T16:18:50.244094", "status": "completed"} tags=[]

FAKE_NEWS_CSV = os.path.join(os.getenv('PROJECT_DIR'), 'data', 'interim', 'GermanFakeNC_interim.csv')
df = pd.read_csv(FAKE_NEWS_CSV)
df.head(20)

# + [markdown] papermill={"duration": 0.026486, "end_time": "2021-09-16T16:18:55.271851", "exception": false, "start_time": "2021-09-16T16:18:55.245365", "status": "completed"} tags=[]
# ## Start exploring the data

# + papermill={"duration": 0.090589, "end_time": "2021-09-16T16:18:55.389333", "exception": false, "start_time": "2021-09-16T16:18:55.298744", "status": "completed"} tags=[]
print('Columns with missing values:')
print(df.isnull().sum())
# -

# Skip rows with NULL values in text and title
df_ = df.dropna(subset=['titel', 'text'])
df_.isnull().sum()

# + papermill={"duration": 0.06018, "end_time": "2021-09-16T16:18:55.532083", "exception": false, "start_time": "2021-09-16T16:18:55.471903", "status": "completed"} tags=[]
# Still NULLS for Date...
df_[df_['Date'].isnull()]
# -

# From the URL we can see that the Date could be 2017/12 so we impute the Date to work with...

df_['Date'].fillna(pd.to_datetime('01/12/2017'), inplace=True)

# + [markdown] papermill={"duration": 0.028934, "end_time": "2021-09-16T16:18:56.231678", "exception": false, "start_time": "2021-09-16T16:18:56.202744", "status": "completed"} tags=[]
# First we check for duplicates in the dataset...

# + papermill={"duration": 1.63057, "end_time": "2021-09-16T16:18:57.891474", "exception": false, "start_time": "2021-09-16T16:18:56.260904", "status": "completed"} tags=[]
print(df_.duplicated(subset=['titel', 'text']).value_counts(normalize=True))
df_[df_.duplicated(subset=['titel', 'text'])]

# + papermill={"duration": 0.689924, "end_time": "2021-09-16T16:18:58.611702", "exception": false, "start_time": "2021-09-16T16:18:57.921778", "status": "completed"} tags=[]
# We drop them...
df_ = df_.drop_duplicates(subset=['titel', 'text']).reset_index()

# + [markdown] papermill={"duration": 0.030143, "end_time": "2021-09-16T16:18:58.672014", "exception": false, "start_time": "2021-09-16T16:18:58.641871", "status": "completed"} tags=[]
# Now lets have a quick look at how many words and characters we have in Titel and Body? 

# + papermill={"duration": 3.286412, "end_time": "2021-09-16T16:19:01.988871", "exception": false, "start_time": "2021-09-16T16:18:58.702459", "status": "completed"} tags=[]
# Count for max character and word length in Titel and Body
print('Max # of char in Title: %i' % df_['titel'].str.len().max())
cnt_lst = []
for i in range(0, len(df_)):
    cnt_lst.append(len(df_['titel'][i].split()))
print('Max # words in Title: %i' % max(cnt_lst))
print('Average # of words in Title: %i' % np.mean(cnt_lst))

print('Max char length of Body: %i' % df_['text'].str.len().max())
cnt_lst = []
for i in range(0, len(df_)):
    cnt_lst.append(len(df_['text'][i].split()))
print('Max # of words in Body: %i' % max(cnt_lst))
print('Average # of words in Body: %i' % np.mean(cnt_lst))

# + [markdown] papermill={"duration": 0.031095, "end_time": "2021-09-16T16:19:02.052285", "exception": false, "start_time": "2021-09-16T16:19:02.021190", "status": "completed"} tags=[]
# Ok, this seams to be quite reasonable for news articles.
#
# ### From what year they actually stem from?

# + papermill={"duration": 0.825199, "end_time": "2021-09-16T16:19:02.909017", "exception": false, "start_time": "2021-09-16T16:19:02.083818", "status": "completed"} tags=[]
df_['Date'] = pd.to_datetime(df_['Date'])
df_['Year'] = df_['Date'].apply(lambda x: x.year)
df_['Year'].plot(kind = 'hist', figsize=(10, 8) )

# + [markdown] papermill={"duration": 0.032314, "end_time": "2021-09-16T16:19:02.973890", "exception": false, "start_time": "2021-09-16T16:19:02.941576", "status": "completed"} tags=[]
# Ok, so the majority of articles were published from 2017 to 2018. We are in 2021 as this EDA was perfromed. So the latest articles are already 3 years old. Could that have an implication on our model we want to train?
# * Topics (and therefore the vocabulary) in the train data could be outdated? (e.g. Nobody talked about covid-19 in 2018!)

# + [markdown] papermill={"duration": 0.031917, "end_time": "2021-09-16T16:19:03.038341", "exception": false, "start_time": "2021-09-16T16:19:03.006424", "status": "completed"} tags=[]
# ## Preprocessing and preparing text features

# + [markdown] papermill={"duration": 0.032357, "end_time": "2021-09-16T16:19:03.104069", "exception": false, "start_time": "2021-09-16T16:19:03.071712", "status": "completed"} tags=[]
# We already know (and expected) from a first look on the text features, that they contain a lot of stopwords. Therefore we take a stopword list from nltk.
#
# Afterwards we ceate a word count matrix with CountVectorizer Object from scikit-learn and play around a bit...

# + papermill={"duration": 0.046642, "end_time": "2021-09-16T16:19:03.184065", "exception": false, "start_time": "2021-09-16T16:19:03.137423", "status": "completed"} tags=[]
# Init a stopword list from nltk and use it as arg in CountVectorizer...
nltk.download('stopwords')
stopword_list = nltk.corpus.stopwords.words('german')
stemmer = nltk.stem.snowball.GermanStemmer(ignore_stopwords=True)


# -

class StemmedCountVectorizer(CountVectorizer):
    def build_analyzer(self):
        analyzer = super(StemmedCountVectorizer, self).build_analyzer()
        return lambda doc: ([stemmer.stem(w) for w in analyzer(doc)])


# Count Vec Titel...

# + papermill={"duration": 1.68061, "end_time": "2021-09-16T16:19:04.897760", "exception": false, "start_time": "2021-09-16T16:19:03.217150", "status": "completed"} tags=[]
count_vec_titel = StemmedCountVectorizer(stop_words=stopword_list,
                                  token_pattern=r'\<KEY>',
                                  max_features=50)
                                  #ngram_range=(2,2)
                                #)
# We first do that on Titel...
count_vec_titel.fit(df_['titel'])
word_counts_in_titel = count_vec_titel.transform(df_['titel']).todense()

# + [markdown] papermill={"duration": 0.032147, "end_time": "2021-09-16T16:19:04.963509", "exception": false, "start_time": "2021-09-16T16:19:04.931362", "status": "completed"} tags=[]
# ### What are the most occuring words?

# + papermill={"duration": 0.057817, "end_time": "2021-09-16T16:19:05.053689", "exception": false, "start_time": "2021-09-16T16:19:04.995872", "status": "completed"} tags=[]
df_titel = pd.DataFrame(word_counts_in_titel, columns=count_vec_titel.get_feature_names())
df_titel_trans = pd.DataFrame(df_titel.T.sum(axis=1), columns=['count'])
df_titel_trans.sort_values(by='count', ascending=False).head(50)

# + [markdown] papermill={"duration": 0.033735, "end_time": "2021-09-16T16:19:05.473716", "exception": false, "start_time": "2021-09-16T16:19:05.439981", "status": "completed"} tags=[]
# Now count vec the body...

# + papermill={"duration": 44.012255, "end_time": "2021-09-16T16:19:49.519822", "exception": false, "start_time": "2021-09-16T16:19:05.507567", "status": "completed"} tags=[]
count_vec_body = StemmedCountVectorizer(stop_words=stopword_list,
                                  token_pattern=r'\b[a-zA-Z]{2,}\b',
                                  max_features=50,
                                  #ngram_range=(2,2)
                                )
count_vec_body.fit(df_['text'])
word_counts_in_body = count_vec_body.transform(df_['text']).todense()

# + papermill={"duration": 0.058799, "end_time": "2021-09-16T16:19:49.612838", "exception": false, "start_time": "2021-09-16T16:19:49.554039", "status": "completed"} tags=[]
df_body = pd.DataFrame(word_counts_in_body, columns=count_vec_body.get_feature_names())
df_body_trans = pd.DataFrame(df_body.T.sum(axis=1), columns=['count'])
df_body_trans.sort_values(by='count', ascending=False).head(20)

StarcoderdataJupyterScriptsDedupFiltered