# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# ## Doing the csv manipulation for January meteorological data

# ### Convert observation minute data to hourly data

import pandas as pd
import numpy as np

dfmin = pd.read_csv('./data/met_data_jan_minute.csv')

dfmin.head()

dfmin.shape

dfmin['datetime'] = pd.to_datetime(dfmin.time)

dfmin.head()

# +
# In this data set dates are missing, from index 17828(+2 in excel file) to 17830(+2 in excel file)

dfmin[dfmin['datetime'].isna()]

# +
# drop the nan rows

dfmin = dfmin.dropna()
dfmin.isna().sum()

# +
# Now we have to add the misssing datetime indices and replace those dates as NaN

import pandas as pd

idx = pd.date_range('2020-01-01 0:00:00', '09-30-2013')

s = pd.Series({'09-02-2013': 2,
               '09-03-2013': 10,
               '09-06-2013': 5,
               '09-07-2013': 1})
s.index = pd.DatetimeIndex(s.index)

s = s.reindex(idx, fill_value=0)
print(s)
# -

dfmin.shape

dfmin = dfmin.set_index(dfmin.datetime)
dfmin = dfmin.drop(['datetime', 'time'], axis=1)

dfmin.head()

dfmin.shape

# +
# Now we fill the missing datetime indices and replace the correspoding column values with NaN

idx = pd.date_range('2021-01-01 00:00:00', '2021-02-01 00:00:00', freq='T')
dfmin = dfmin.reindex(idx, fill_value=np.nan)
# -

dfmin.shape

dfmin.isna().sum()

dfmin[dfmin['obs_lwlower'].isna()]

dfhour = dfmin.resample('H').mean()

dfhour.head()

dfhour = dfhour.iloc[:-1,:]

dfhour

dfhour.to_csv('./data/met_data_jan_hour.csv')







# +
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
# %matplotlib inline
import seaborn as sns
import datetime

# Use seaborn style defaults and set default figure size
sns.set_style("whitegrid")
# -

dfmod = pd.read_csv('data/aqi_data_mod.csv')
dfobs = pd.read_csv('data/aqi_data_obs.csv')

dfmod['datetime'] = pd.to_datetime(dfmod.date) + pd.to_timedelta(dfmod.hour)
dfobs['datetime'] = pd.to_datetime(dfobs.date) + pd.to_timedelta(dfobs.hour)

dfmod = dfmod.drop(['date', 'hour'], axis=1)
dfobs = dfobs.drop(['date', 'hour'], axis=1)

dfmod = dfmod.set_index(dfmod.datetime)
dfobs = dfobs.set_index(dfobs.datetime)

# +
# Do interpolatoin of the observed data

dfobs = dfobs.interpolate(method = 'time')
# -

import sys
sys.path.append('../functions/')

print(sys.path)

import dplot

dplot.plot(dfmod, dfobs, 'mod_PM2.5', 'obs_PM2.5', mod_stdev='mod_PM2.5_stdev', obs_stdev='obs_PM2.5_stdev')

dfmod.head(25)



StarcoderdataJupyterScriptsDedupFiltered