# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# import library and url to load LatLong from
import pandas as pd
import requests
import time

# + pycharm={"name": "#%%\n"}
# Downloading PRECTOT,QV2M,PS,TS,T2MDEW,T2M,WS50M,WS10M,T2MWET,T2M_RANGE,RH2M,KT,CLRSKY_SFC_SW_DWN,ALLSKY_SFC_SW_DWN,ALLSKY_SFC_LW_DWN

# Actual URL --> https://power.larc.nasa.gov/cgi-bin/v1/DataAccess.py?&request=execute&identifier=SinglePoint&parameters=PRECTOT,QV2M,PS,TS,T2MDEW,T2M,WS50M,WS10M,T2MWET,T2M_RANGE,RH2M,KT,CLRSKY_SFC_SW_DWN,ALLSKY_SFC_SW_DWN,ALLSKY_SFC_LW_DWN&startDate=19810101&endDate=20201231&userCommunity=SSE&tempAverage=DAILY&outputList=CSV&lat=41.1560&lon=-100.2092

#base url to get the data from
base_url = 'https://power.larc.nasa.gov/cgi-bin/v1/DataAccess.py?&request=execute&identifier=SinglePoint&parameters=PRECTOT,QV2M,PS,TS,T2MDEW,T2M,WS50M,WS10M,T2MWET,T2M_RANGE,RH2M,KT,CLRSKY_SFC_SW_DWN,ALLSKY_SFC_SW_DWN,ALLSKY_SFC_LW_DWN&startDate=19810101&endDate=20201231&userCommunity=SSE&tempAverage=DAILY&outputList=CSV&lat='

#Import Lat Long from another CSV. This information will be used to request data from NASA app
LatLong = pd.read_csv('/home/rob-rog/PycharmProjects/Colab_Notebooks/Unit 2 - Linear Models/Sprint_4_Build_Precipitation_Prediction/LatLong/average-latitude-longitude-countries.csv')
LatLong.head()

#make get request and make json file
r = requests.get(base_url+f"{LatLong['Latitude'][0]}&lon={LatLong['Longitude'][0]}")
json = r.json()

#instantiate the DataFrame
df = pd.DataFrame()

# merge function
def merge(df1, df2):
    merged = df1.merge(df2, on='date')
    return merged

#define function to convert json data to DataFrame
def to_dataframe(json_data):
    list_features = r.json()['features']
    len(list_features)

    dict_features = dict(list_features[0])
    type(dict_features)

    # dict_features_geo_coordinates = \
    dict_features_geo = dict_features['geometry']
    dict_features_geo_coodinates = dict_features_geo['coordinates']
    lat = dict_features_geo_coodinates[0]
    long = dict_features_geo_coodinates[1]
    elevation = dict_features_geo_coodinates[2]
    # Subsetting the json data to reach to the features
    dict_features_prop = dict_features['properties']
    dict_features_prop_para = dict_features_prop['parameter']
    dict_features_prop_para_prec = dict_features_prop_para['PRECTOT']
    dict_features_prop_para_qv2m = dict_features_prop_para['QV2M']
    dict_features_prop_para_surfp = dict_features_prop_para['PS']
    dict_features_prop_para_ts = dict_features_prop_para['TS']
    dict_features_prop_para_t2mdew = dict_features_prop_para['T2MDEW']
    dict_features_prop_para_t2m = dict_features_prop_para['T2M']
    dict_features_prop_para_ws10m = dict_features_prop_para['WS10M']
    dict_features_prop_para_ws50m = dict_features_prop_para['WS50M']
    dict_features_prop_para_t2mwet = dict_features_prop_para['T2MWET']
    dict_features_prop_para_t2mrange = dict_features_prop_para['T2M_RANGE']
    dict_features_prop_para_rh2m = dict_features_prop_para['RH2M']
    dict_features_prop_para_kt = dict_features_prop_para['KT']
    dict_features_prop_para_clrskysw = dict_features_prop_para['CLRSKY_SFC_SW_DWN']
    dict_features_prop_para_allskysw = dict_features_prop_para['ALLSKY_SFC_SW_DWN']
    dict_features_prop_para_allskylw = dict_features_prop_para['ALLSKY_SFC_LW_DWN']
    # making a dataframe for each feature
    df_prec = pd.DataFrame(list(dict_features_prop_para_prec.items()), columns=['date', 'precipitation'])
    df_qv2m = pd.DataFrame(list(dict_features_prop_para_qv2m.items()), columns=['date', 'specific_humidity'])
    df_ps = pd.DataFrame(list(dict_features_prop_para_surfp.items()), columns=['date', 'surface_pressure'])
    df_ts = pd.DataFrame(list(dict_features_prop_para_ts.items()), columns=['date', 'skin_temperature'])
    df_t2mdew = pd.DataFrame(list(dict_features_prop_para_t2mdew.items()), columns=['date', 'dew_frost'])
    df_t2m = pd.DataFrame(list(dict_features_prop_para_t2m.items()), columns=['date', 'temperature2m'])
    df_ws10m = pd.DataFrame(list(dict_features_prop_para_ws10m.items()), columns=['date', 'windspeed10m'])
    df_ws50m = pd.DataFrame(list(dict_features_prop_para_ws50m.items()), columns=['date', 'windspeed50m'])
    df_t2mwet = pd.DataFrame(list(dict_features_prop_para_t2mwet.items()), columns=['date', 'wet_bulb_temp'])
    df_t2mrange = pd.DataFrame(list(dict_features_prop_para_t2mrange.items()), columns=['date', 'temp_range'])
    df_rh2m = pd.DataFrame(list(dict_features_prop_para_rh2m.items()), columns=['date', 'rel_humidity'])
    df_kt = pd.DataFrame(list(dict_features_prop_para_kt.items()), columns=['date', 'clearness_index'])
    df_clrskysw = pd.DataFrame(list(dict_features_prop_para_clrskysw.items()), columns=['date', 'clear_sky_insolation'])
    df_allskysw = pd.DataFrame(list(dict_features_prop_para_allskysw.items()), columns=['date', 'all_sky_insolation'])
    df_allskylw = pd.DataFrame(list(dict_features_prop_para_allskylw.items()), columns=['date', 'radiative_flux'])

    # merging all dataframe in one
    merge1 = merge(df_prec, df_qv2m)
    merge2 = merge(df_ps, df_ts)
    merge3 = merge(df_t2mdew, df_t2m)
    merge4 = merge(df_ws10m, df_ws50m)
    merge5 = merge(df_t2mwet, df_t2mrange)
    merge6 = merge(df_rh2m, df_kt)
    merge7 = merge(df_clrskysw, df_allskysw)
    merge12 = merge(merge1, merge2)
    merge34 = merge(merge3, merge4)
    merge56 = merge(merge5, merge6)
    merge78 = merge(merge7, df_allskylw)
    merge1234 = merge(merge12, merge34)
    merge5678 = merge(merge56, merge78)
    merge_final = merge(merge1234, merge5678)

    merge_final['lat'] = lat
    merge_final['long'] = long
    merge_final['elevation'] = elevation

    #round to two decimals
    merge_final['lat'] = merge_final['lat'].round(2)
    merge_final['long'] = merge_final['long'].round(2)
    merge_final['elevation'] = merge_final['elevation'].round(2)
    merge_final['country'] = country
    merge_final['country_code'] = country_code

    # date to datetime object
    merge_final['date'] = pd.to_datetime(merge_final['date'], infer_datetime_format=True)

    #rearrange
    merge_final = merge_final[['date','country', 'country_code', 'lat', 'long', 'elevation', 'specific_humidity', 'surface_pressure', 'skin_temperature', 'dew_frost', 'temperature2m', 'windspeed10m', 'windspeed50m', 'wet_bulb_temp', 'temp_range', 'rel_humidity', 'clearness_index', 'clear_sky_insolation', 'all_sky_insolation', 'radiative_flux', 'precipitation']]
    return merge_final

print('Scraping Data from NASA Servers. Initial DF shape', df.shape)
start_time = time.time()

# loop to iterate the lat long information and get request from NASA app
for i in range(0,240):
    r = requests.get(base_url+f"{LatLong['Latitude'][i]}&lon={LatLong['Longitude'][i]}")
    country = LatLong['Country'][i]
    country_code = LatLong['ISO 3166 Country Code'][i]
    print('Requesting Surface Pressure data for ',country,' from NASA servers')
    json = r.json()
    df1 = to_dataframe(json)
    df = pd.concat([df, df1])
    now = time.time()
    print('Success DataFrame shape is ', df.shape, 'Time Elapsed ',(now - start_time) )

#export to CSV file
df.to_csv('global_nasa_weather_data.csv')
df

# + [markdown] pycharm={"name": "#%% md\n"}
# ## Choosing the target
#
# The target is precipitation
#
# ## Is your problem regression or classification?
#
# Regression problem
#
# ## How is your target distributed?
#
# The target is a quantitative data and by looking at the distribution plot we can see the data is skewed towards the right
#
# ## Regression: Is the target right-skewed? If so, you may want to log transform the target.
#
# There are only 376 rows out of approximately 3.5 Million observations that is making the data to be skewed to the right.
#
# ## Choose your evaluation metric(s). Regression: Will you use mean absolute error, root mean squared error, R^2, or other regression metrics?
#
# R squared
#
# ## Choose which observations you will use to train, validate, and test your model.
#
# A train test split will be required
#
# ## Begin to choose which features, if any, to exclude. Would some features "leak" future information?
#
# Could not find any feature that may leak future information.
#
#
#

# + pycharm={"name": "#%%\n"}
import seaborn as sns

sns.distplot(df['precipitation'])

# + pycharm={"name": "#%%\n"}
condition = df['precipitation'] > 100
df[condition]

StarcoderdataJupyterScriptsDedupFiltered