# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

# Enabling reloading extension to use external python files
# %load_ext autoreload
# %autoreload 2

import pandas as pd
import numpy as np
import os
from datetime import datetime, timedelta
import columns
import datasets
import seasons
import directory_utils

# +
_MIN_DAY_VALUE = 1
_MAX_DAY_VALUE = 366
_DAYS_RANGE = range(_MIN_DAY_VALUE - 1, _MAX_DAY_VALUE + 1)
#print("Days range: " + str(_DAYS_RANGE))

_MIN_HOUR_VALUE = 0
_MAX_HOUR_VALUE = 2400
_HOURS_RANGE = range(_MIN_HOUR_VALUE - 1, _MAX_HOUR_VALUE + 1)
#print("Hours range: " + str(_HOURS_RANGE))

_MIN_YEAR_VALUE = 1997
_MAX_YEAR_VALUE = 2016
_YEARS_RANGE = range(_MIN_YEAR_VALUE - 1, _MAX_YEAR_VALUE + 1)
#print("Years range: " + str(_YEARS_RANGE))
# -

data_frame = pd.read_csv(datasets.MERGED_CSVS__DATA_SET_111_PATH, encoding=datasets.DEFAULT_CSV_ENCODING)

original_shape = data_frame.shape
original_shape

data_frame.head(2)

data_frame.describe()


# ### Removing invalid days, hours and years

def is_in_range(range, value):
    return value in range
def is_not_in_range(range, value):
    return value not in range


_days_not_in_range_filter = data_frame[columns.JULIAN_DAY].map(lambda value: is_not_in_range(_DAYS_RANGE, value))
data_frame[_days_not_in_range_filter][columns.JULIAN_DAY].unique()

_days_in_range_filter = data_frame[columns.JULIAN_DAY].map(lambda value: is_in_range(_DAYS_RANGE, value))
data_frame = data_frame[_days_in_range_filter]

_years_not_in_range_filter = data_frame[columns.YEAR].map(lambda value: is_not_in_range(_YEARS_RANGE, value))
data_frame[_years_not_in_range_filter][columns.YEAR].unique()

_years_in_range_filter = data_frame[columns.YEAR].map(lambda value: is_in_range(_YEARS_RANGE, value))
data_frame = data_frame[_years_in_range_filter]

# +
_hours_not_in_range_filter = data_frame[columns.HOUR_MINUTE].map(lambda value: is_not_in_range(_HOURS_RANGE, value))

data_frame[_hours_not_in_range_filter][columns.HOUR_MINUTE].unique()
# -

_hours_in_range_filter = data_frame[columns.HOUR_MINUTE].map(lambda value: is_in_range(_HOURS_RANGE, value))
data_frame = data_frame[_hours_in_range_filter]

data_frame.shape

# ### Removing null values

data_frame.isnull().sum()

data_frame.shape

data_frame.dropna().shape

data_frame = data_frame.dropna()

data_frame.isnull().sum()

data_frame.shape

# ### Removing days with value "0"

data_frame[data_frame[columns.JULIAN_DAY]==0]

data_frame = data_frame[data_frame[columns.JULIAN_DAY]!=0]

data_frame.shape

# ### Removing data from 222

data_frame[columns.CODE].unique()

data_frame[data_frame[columns.CODE]!=111].head()

data_frame = data_frame[data_frame[columns.CODE]==111]

data_frame.shape

# ### Removing invalid Air Temperature measurements

_MIN_AIR_TEMPERATURE=-2
_MAX_AIR_TEMPERATURE=45

data_frame[columns.TEMPERATURE].describe()

data_frame_without_invalid_air_temp = data_frame.loc[(data_frame[columns.TEMPERATURE] > _MIN_AIR_TEMPERATURE) & (data_frame[columns.TEMPERATURE] < _MAX_AIR_TEMPERATURE), :]
data_frame_without_invalid_air_temp[columns.TEMPERATURE].describe()

data_frame.shape[0] - data_frame_without_invalid_air_temp.shape[0]

data_frame = data_frame_without_invalid_air_temp

# ### Removing invalid Air moisture measurements

_MIN_AIR_MOISTURE=0
_MAX_AIR_MOISTURE=100

data_frame[columns.HUMIDITY].describe()

data_frame_without_invalid_air_mois = data_frame.loc[(data_frame[columns.HUMIDITY] >= _MIN_AIR_MOISTURE) & (data_frame[columns.HUMIDITY] <= _MAX_AIR_MOISTURE), :]
data_frame_without_invalid_air_mois[columns.HUMIDITY].describe()

data_frame.shape[0] - data_frame_without_invalid_air_mois.shape[0]

data_frame = data_frame_without_invalid_air_mois

# ### Removing invalid Air pressure measurements

_MIN_AIR_PRESSURE=822
_MAX_AIR_PRESSURE=1030

data_frame[columns.PRESSURE].describe()

data_frame_without_invalid_air_press = data_frame.loc[(data_frame[columns.PRESSURE] >= _MIN_AIR_PRESSURE) & (data_frame[columns.PRESSURE] <= _MAX_AIR_PRESSURE), :]
data_frame_without_invalid_air_press[columns.PRESSURE].describe()

data_frame.shape[0] - data_frame_without_invalid_air_press.shape[0]

data_frame = data_frame_without_invalid_air_press

# ### Removing invalid Wind Directionmeasurements

_MIN_WIND_DIRECTION=0
_MAX_WIND_DIRECTION=360

data_frame[columns.WIND_DIRECTION].describe()

data_frame_without_invalid_wind_dir = data_frame.loc[(data_frame[columns.WIND_DIRECTION] >= _MIN_WIND_DIRECTION) & (data_frame[columns.WIND_DIRECTION] <= _MAX_WIND_DIRECTION), :]
data_frame_without_invalid_wind_dir[columns.WIND_DIRECTION].describe()

data_frame.shape[0] - data_frame_without_invalid_wind_dir.shape[0]

# +
# I'm commenting this for now to avoid removing valid temperature and pression measurements.
#data_frame = data_frame_without_invalid_wind_dir
# -

# ### Results after cleaning

# +
original_amount_of_lines = original_shape[0]
amount_of_lines_after_cleaning = data_frame.shape[0]
amount_of_lines_removed = original_amount_of_lines - amount_of_lines_after_cleaning

percentage_of_lines_removed = 100 * amount_of_lines_removed / original_amount_of_lines

percentage_of_lines_removed = round(percentage_of_lines_removed, 2)

print(str(amount_of_lines_removed) + " observations were removed what is " + str(percentage_of_lines_removed) + " of the dataset")

# -

# ### Creating the date column

year_mapping = data_frame[columns.YEAR].map(lambda year : datetime(year, 1, 1))
time_delta_mapping = data_frame[columns.JULIAN_DAY].map(lambda day : timedelta(day-1))
data_frame[columns.DATE] = year_mapping + time_delta_mapping

# ### Saving CSV

directory_utils.create_if_doesnt_exist(datasets.FIRST_CLEAN_DIRECTORY_PATH)
data_frame.to_csv(datasets.FIRST_CLEAN_DATA_SET_111_PATH, index = False, encoding=datasets.DEFAULT_CSV_ENCODING)



StarcoderdataJupyterScriptsDedupFiltered