# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # COVID19 Global Forecasting (Week 5)

# ## Problem Statement
#
# In this challenge, you will be predicting the daily number of confirmed COVID19 cases in various locations across the world, as well as the number of resulting fatalities, for future dates. This latest challenge includes US state county data.

# ## Data 
#
# * This evaluation data for this competition comes from John Hopkins CSSE, which is uninvolved in the competition.
# * See their README for a description of how the data was collected.
# * They are currently updating the data daily.

# ## Solution

# ### Libraries

# +
# Standard libraries 

import pandas as pd 
import numpy as np 
import matplotlib.pyplot as plt
import seaborn as sns 
import time




# Libraries for EDA and preprocessing 
import datetime as dt
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import MinMaxScaler

# libraies for Model 
from xgboost import XGBRegressor
from sklearn.model_selection import GridSearchCV

# libraries for Evaluations
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import StratifiedKFold

# +
### Load data to variables 

train_data = pd.read_csv('Datasets/covid19-global-forecasting-week-5/train.csv')
test_data = pd.read_csv('Datasets/covid19-global-forecasting-week-5/test.csv')

# +
# print the first 2 rows of train_data 

train_data.head(2)

# +
# print the first two rows of test_data

test_data.head(2)
# -

# ### EDA and preprocessing

# +
# Information about train_data 

train_data.info()
# -

# **Our train_data contains 9 features which contains 1 int value, 5 object values, and 2 float values**

# +
# size of our data set 

train_data.shape # Contains 754934 data

# +
# check for any null values in our train data set 

train_data.isna().sum()
# -

# **We can see that our train_data has null values in following columns with respective numbers:**
#     
# * County      with      69760
# * Province_State   with 40766

# +

# lets check the information about test_data 

test_data.info()
# -

# **Test data contains all the columns  in train_data except for Target_values, It contains float64(1), int64(2), object(5)**

# +
# find if there is any null value in the test data set 

test_data.isna().sum()
# -

# **Test data also contains 28800 null values of county and 16830 values of Provice_state**

# +
# Remove the null value data from our train_data set 

train_data.drop(['Id', 'County', 'Province_State'], axis = 1, inplace = True)
train_data.head()

# -

# Remove the null value data from our test_data set 
test_data.drop(['ForecastId','County', 'Province_State'], axis =1, inplace = True )
test_data.head()

# +
# Check if there any null values present in train_data 

train_data.isna().sum()
# -

train_data.describe()

# +
# lets count the target values 

train_data['Target'].value_counts()

# +

fig, ax= plt.subplots(figsize = (20,10))
ax.pie(x=train_data.groupby(by=["Country_Region"])["TargetValue"].sum(), startangle=90)
label =train_data['Country_Region'].unique()
plt.title("Countries with COVID-19 Cases", fontsize = 24) 
fig.legend(label) 
plt.tight_layout() 
plt.show()
# -

# visualization of comfirmed and fatal cases
fig, ax= plt.subplots(figsize = (20,10))
ax.pie(x=train_data.groupby(by=["Target"])["TargetValue"].sum(), autopct='%1.1f%%', startangle=90)
label =train_data['Target'].unique()
plt.title("Comfirmed and Fatal COVID-19 Cases", fontsize = 24) 
fig.legend(label) 
plt.tight_layout() 
plt.show()

sns.countplot(x ='Target', data = train_data);

# Finding the latest case in the countries
last_date = train_data.Date.max()
df=train_data[train_data["Date"]==last_date]
df

# calculating the sum of targetvalue of country region
df=df.groupby(by=["Country_Region"],as_index=False)["TargetValue"].sum()
df

# 5 countries with large target value.
countries = df.nlargest(5,"TargetValue")
countries

cases=train_data.groupby(by=["Date","Country_Region"],as_index=False)["TargetValue"].sum()
cases

cases=cases.merge(countries,on="Country_Region")
cases

# visualization of the target value with the country
plt.figure(figsize=(15,10))
sns.set(style="darkgrid")
sns.lineplot(x="Date",y="TargetValue_x",hue="Country_Region",data=cases);

# finding the correlation 
train_data.corr()

train_data.drop(["Target"],inplace=True,axis=1)
test_data.drop(["Target"],inplace=True,axis=1)
train_data

# +
# transform the country_region variable

lab = LabelEncoder()
train_data['Country_Region'] = lab.fit_transform(train_data['Country_Region'])
train_data.head()
# -

test_data['Country_Region'] = lab.fit_transform(test_data['Country_Region'])

test_data.head()

train_data.Date=train_data.Date.apply(lambda x:x.split("-"))

test_data.Date=test_data.Date.apply(lambda x:x.split("-"))


# +
# split Date into month and day 

def monthday(data):
    month = []
    day= []
    for date in data.Date:
        month.append(int(date[1]))
        day.append(int(date[2]))
    data['month'] = month
    data['day'] = day
    data = data.drop('Date', axis =1 )
    return data 


# -

train_data = monthday(train_data)
test_data = monthday(test_data)
train_data.head()

# ### Modelling

# +
# Split data into  y
scaler = MinMaxScaler()


y = train_data['TargetValue'].values
train_data.drop(['TargetValue'], axis = 1, inplace = True)
# -

train_data.head()

# Transforming features by scaling each feature to a given range.
X = scaler.fit_transform(train_data)
X

# initialize the xgb
xgb = XGBRegressor()

# +
# perform the measure
performance = cross_val_score(xgb, X, y, cv=10, scoring ='neg_mean_absolute_error', n_jobs = -1)

mae = -performance
# -

mae

mae.mean()

# +
# transform the test data 

test_data= scaler.fit_transform(test_data)
test_data
# -

xgb.fit(X,y)
prediction = xgb.predict(test_data)
prediction = np.around(prediction)
prediction

xgb_1500=XGBRegressor(n_estimators=1500,max_depth=15)

xgb_1500.fit(X,y)


prediction=xgb_1500.predict(test_data)
prediction=np.around(prediction)
prediction

submission=pd.read_csv("Datasets/covid19-global-forecasting-week-5/submission.csv")
submission.head()

test_copy=pd.read_csv('Datasets/covid19-global-forecasting-week-5/test.csv')

output = pd.DataFrame({'Id': test_copy.ForecastId  , 'TargetValue': prediction})
output.head()

a=output.groupby(['Id'])['TargetValue'].quantile(q=0.05).reset_index()
b=output.groupby(['Id'])['TargetValue'].quantile(q=0.5).reset_index()
c=output.groupby(['Id'])['TargetValue'].quantile(q=0.95).reset_index()

a.columns=['Id','q0.05']
b.columns=['Id','q0.5']
c.columns=['Id','q0.95']
a=pd.concat([a,b['q0.5'],c['q0.95']],1)
a['q0.05']=a['q0.05']
a['q0.5']=a['q0.5']
a['q0.95']=a['q0.95']

sub=pd.melt(a, id_vars=['Id'], value_vars=['q0.05','q0.5','q0.95'])
sub['variable']=sub['variable'].str.replace("q","", regex=False)
sub['ForecastId_Quantile']=sub['Id'].astype(str)+'_'+sub['variable']
sub['TargetValue']=sub['value']
sub=sub[['ForecastId_Quantile','TargetValue']]
sub.reset_index(drop=True,inplace=True)
sub.to_csv("submission.csv",index=False)
sub.head()

StarcoderdataJupyterScriptsDedupFiltered