# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import numpy as np
import pandas as pd
from matplotlib.lines import Line2D
import matplotlib.pyplot as plt
import matplotlib as mpl
import seaborn as sns

df_covid_stats = pd.read_excel('covid_stats.xlsx')

df_covid_stats.head(5)

df_covid_stats.info()

df = pd.read_pickle('india_dataset_Mar_Apr.pkl')

df.published_at[:5]

df_news_vol = df[['published_at', 'title']].groupby(by='published_at').count().reset_index()
df_news_vol.columns = ['published_at', 'News Volume']

df_news_vol.head(5)

df_news_vol.info()

df_news_vol['published_at'] = pd.to_datetime(df_news_vol['published_at'])
df_news_vol.head()

df_merged = pd.merge(df_news_vol, df_covid_stats, left_on='published_at', right_on='Date')
df_merged.head()

df_merged.drop(columns='published_at', inplace=True)

df_merged.set_index('Date', inplace=True)

df_merged.head()

df_merged.info()

import datetime
custom_text = "Generated by NRoy@https://github.com/royn5618 on " + str(datetime.datetime.now().date())
custom_text

# ### Time Series Plot

# +
columns = ['News Volume', 'Total Cases', 'Total Death']
colors = ['black', '#007acc', '#4db8ff']

plt.style.use('seaborn-darkgrid') 

# Draw plot
fig, ax = plt.subplots(figsize=(16,10), dpi= 80)

for i, column in enumerate(columns):
    ax.vlines(x=df_merged.index, ymin=0, ymax=df_merged[column], color='grey', linewidth=1, alpha=0.5)
    ax.scatter(x=df_merged.index, y=df_merged[column], s=75, color=colors[i])

# Title, Label, Ticks and Ylim
ax.set_title('News Volume, Covid Cases and Deaths', fontdict={'size':22})
ax.set_ylabel('News Volume')
ax.set_xticks(df_merged.index)
ax.set_xticklabels(df_merged.index.date, rotation=60, fontdict={'horizontalalignment': 'right', 'size':12})
ax.set_ylim(bottom=0)


# Setting legend
legend_elements = [Line2D([0], [0], marker='o', color='black', label='News Volume', markersize=10),
                   Line2D([0], [0], marker='o', color='#007acc', label='Total Cases', markersize=10),
                   Line2D([0], [0], marker='o', color='#4db8ff', label='Total Death', markersize=10)]
ax.legend(handles=legend_elements, loc='upper left')
plt.text(737512.5, 920, custom_text, size='medium', color='black')

plt.show()
# -

# ### Counting how many times corona/covid occured per day in titles

df.head(2)

# +
from nltk.corpus import stopwords
import string
STOPWORDS = set(stopwords.words('english'))

def clean_tokens(token_sequences):
    cleaned_token_sequences = []
    for token_list in token_sequences:
        tokens = [x.strip().lower() for x in token_list if x != 'char' and x.strip().lower() not in STOPWORDS 
                      and x.strip() not in string.punctuation and x.strip() not in ['’', '‘' , '–', '\'s', 'n\'t' , '’', '``', "''"] 
                      and '…' not in x and '+' not in x and len(x) != 1]
        tokens = ['covid-19' if x == 'covid' else x for x in tokens]
        tokens = ['cor<PASSWORD>' if x == 'corona' else x for x in tokens]
        tokens = ['government' if x == 'govt' else x for x in tokens]
        tokens = ['modi' if x == 'namo' else x for x in tokens]
        cleaned_token_sequences.append(tokens)
    return cleaned_token_sequences


# -

def word_count_title(x, words_list):
    title_tokens = x[0]
    if len(words_list) == 1:
        return title_tokens.count(words_list[0])
    elif len(words_list) >1 :
        return sum([title_tokens.count(each_word) for each_word in words_list])
    else:
        raise Exception("Recheck Wordlist")


covid_words_list = ['covid-19','corona', 'coronavirus']
central_government_list = ['government', 'modi', 'pm']
healthcare_list = ['doctor', 'healthcare', 'vaccine', 'cure', 'hospital', 'test', 'testing']

df['title_cleaned'] = df.title_normalized.apply(lambda x: clean_tokens(x))

df['Covid Count'] = df.title_cleaned.apply(lambda x: word_count_title(x, covid_words_list))
df['Central Gov Count'] = df.title_cleaned.apply(lambda x: word_count_title(x, central_government_list))
df['Healthcare Count'] = df.title_cleaned.apply(lambda x: word_count_title(x, healthcare_list))

df['Covid Count'].value_counts()

df['Central Gov Count'].value_counts()

df['Healthcare Count'].value_counts()

df_bow_extended = df[['Covid Count', 'Central Gov Count', 'Healthcare Count', 'published_at']].groupby(by='published_at').sum().reset_index()
df_bow_extended

# ### Joining this information with df_merged

df_bow_extended['published_at'] = pd.to_datetime(df_bow_extended['published_at'])
df_merged_extended = pd.merge(df_merged, df_bow_extended, left_on='Date', right_on='published_at')
df_merged_extended.head()

# +
plt.close()
columns = ['News Volume','Total Cases', 'Total Death']
columns_bow = ['Covid Count', 'Healthcare Count', 'Central Gov Count']
colors = ['black', '#007acc', '#4db8ff']
colors_bow = ['#cc3300', '#0000e6', '#ff8533']
markers = ['--', ':']

plt.style.use('seaborn-darkgrid') 
fig, ax = plt.subplots(figsize=(15, 10), dpi=100)

for i, column in enumerate(columns):
    ax.plot('published_at', column, '--', data=df_merged_extended, color=colors[i], alpha=0.5)
    

for i, column in enumerate(columns_bow):
    ax.plot('published_at', column, data=df_merged_extended, color=colors_bow[i], linewidth=2)

plt.title('Covid Cases and Deaths, News Volume and BOW Features from 10th March to 10th April, 2020')
plt.xticks(rotation=45)
plt.ylim(0, 900)
plt.xlim(min(df['published_at']), max(df['published_at']))
print(ax.get_xlim())
print(ax.get_ylim())
plt.text(737520.5, 310, 'News Volume', size='medium', color='black')
plt.text(737513.5, 870, custom_text, size='medium', color='black')
plt.legend()

plt.show()
# -

df_merged_extended.head(5)

# ### Let's Play a bit wit correlations

plt.scatter(df_merged_extended['News Volume'], df_merged_extended['Covid Count'])
plt.xlabel('News Volume')
plt.ylabel('Covid Count')

sns.pairplot(df_merged_extended)

corr = df_merged_extended.corr()
corr

# Use this code for Mask
#mask_matrix=np.zeros_like(corr)
#mask_matrix[np.triu_indices_from(mask_matrix)] = True
#mask_matrix = 1 - mask_matrix.T
#sns.heatmap(corr, annot=True, vmin=-1, vmax=1, center=0, square=True, cmap='Spectral', mask = mask_matrix)
plt.style.use('default')
plt.figure(figsize=[10,10])
sns.heatmap(corr, annot=True, vmin=-1, vmax=1, center=0, square=True, cmap='Spectral', annot_kws={"size": 12}, linewidths=1, linecolor='white')
plt.title("Heatmap to Assess Correlation between Extracted Features")
plt.text(2,7.7, custom_text, size='small')

# +
from scipy.stats import pearsonr

def assess_corr(col1, col2):
    r, p = pearsonr(col1, col2)
    return r, p


# -

df_merged_extended.fillna(value=0, inplace=True)
df_merged_extended.set_index('published_at', inplace=True)

df_stat_sig = pd.DataFrame(columns=df_merged_extended.columns, index=df_merged_extended.columns)
df_stat_sig

df_stat_sig['Total Cases']['News Volume']

# +
for col1 in df_merged_extended.columns:
    for col2 in df_merged_extended.columns:
        #print("{} and {}".format(col1, col2))
        r, p = assess_corr(df_merged_extended[col1],df_merged_extended[col2])
        if p < 0.01:
            df_stat_sig[col1][col2] = "{:.3f}".format(p) + "***"
        elif p < 0.02:
            df_stat_sig[col1][col2] = "{:.3f}".format(p) + "**"
        elif p < 0.05:
            df_stat_sig[col1][col2] = "{:.3f}".format(p) + "*"
        else:
            df_stat_sig[col1][col2] = "{:.3f}".format(p)

df_stat_sig
# -

# # Thanks for visiting

# ** Still in Progress ** 

StarcoderdataJupyterScriptsDedupFiltered