# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import pandas as pd
import gc

# # 1. generate file with count of meter_reading equals 0

# count meter_reading equals 0 per site_id, timestamp and meter

# +
# %%time

train_df3= pd.read_csv("train.csv")
building_df = pd.read_csv("building_metadata.csv")
train_df3 = train_df3.merge(building_df[['site_id', 'building_id']], on='building_id', how='left')
toto = train_df3.groupby(['site_id','timestamp', 'meter']).meter_reading.transform(lambda x: x[x.eq(0)].size)
train_df3['cnt_tmp_meter_siteid'] = toto
train_df3.to_feather('train_with_cnt_per_tmp_meter.feather')
# -



# # 2. generate file with hot encoding of meters per building

# a table listing all the building with 4 columns corresponding to the 4 meters. if the building has a partical meter, the value is 1 else it is 0.

# +
# %%time

train_df= pd.read_csv("train.csv")
train_df = train_df.set_index(['building_id', 'meter', 'timestamp'])

train_df = train_df.unstack(1)
train_df.fillna(-1)
train_df2 = train_df
train_df2.columns = ['meter_reading_0','meter_reading_1','meter_reading_2','meter_reading_3',]
train_df2.reset_index(inplace=True)

for col in ['meter_reading_0','meter_reading_1','meter_reading_2','meter_reading_3',]:
    train_df2[col].loc[~train_df2[col].isnull()]=1
    train_df2[col].loc[train_df2[col].isnull()]=0

del train_df2['timestamp']

train_df3 = train_df2.drop_duplicates(subset=['building_id', 'meter_reading_0','meter_reading_1','meter_reading_2','meter_reading_3',], keep="last").reset_index(drop=True)

train_df3['meter_reading_0'] = train_df3.groupby('building_id')['meter_reading_0'].transform(max)
train_df3['meter_reading_1'] = train_df3.groupby('building_id')['meter_reading_1'].transform(max)
train_df3['meter_reading_2'] = train_df3.groupby('building_id')['meter_reading_2'].transform(max)
train_df3['meter_reading_3'] = train_df3.groupby('building_id')['meter_reading_3'].transform(max)

train_df3 = train_df3.drop_duplicates(subset=['building_id', 'meter_reading_0','meter_reading_1','meter_reading_2','meter_reading_3',], keep="last").reset_index(drop=True)

train_df3.to_feather('building_all_meters.feather')
# -

# # 3. cleanup train dataset to remove "vertical lines"

# in his kernel, Ganfear shows the raw data https://www.kaggle.com/ganfear/missing-data-and-zeros-visualized and visualize when the target (meter_reading) is missing or 0. We can clearly see that there are some vertical blue lines: these lines most likely mean that the 0s were added and it is even more likely that they were added when these vertical lines happen on several meters, at the same time and the same site. The goal of the code below is to remove most of them.
# Note: a single groupby takes 3.5 hours to be calculated.

# +
# %%time

building_df = pd.read_csv("building_metadata.csv")
train = pd.read_csv("train.csv")

building_df['cnt_building_per_site'] = building_df.groupby(['site_id']).building_id.transform(lambda x: x.size)
train = train.merge(building_df[['building_id', 'cnt_building_per_site', 'site_id']], on='building_id', how='left')

print('starting')
train['cnt_mreadEQ0_per_tmp_site'] = train.groupby(['timestamp','site_id']).meter_reading.transform(lambda x: x[x.eq(0)].size)
print('done 1')
train['cnt_mreadEQ0_per_tmp_site_building'] = train.groupby(['timestamp','site_id','building_id']).meter_reading.transform(lambda x: x[x.eq(0)].size)
print('done 2')
train['sum_meter_reading'] = train.groupby(['building_id', 'meter']).meter_reading.transform('sum')
print('done 3')

print(train.shape)
df_train2 = pd.read_feather('train_with_cnt_per_tmp_meter.feather')
train['cnt_tmp_meter_siteid'] = df_train2.cnt_tmp_meter_siteid    ### groupby('site_id','timestamp', 'meter') x[x.eq(0)].size)
del df_train2

# +
print(train.shape)

train = train.query('not (building_id <= 104 & meter == 0 & timestamp <= "2016-05-21")')
print(train.shape)

train = train[~(((train.cnt_mreadEQ0_per_tmp_site_building>2) & (train.meter_reading==0)))]
print(train.shape)

train = train[~(((train.cnt_mreadEQ0_per_tmp_site_building==2) & (train.meter_reading==0) & (train.meter==0)))]
print(train.shape)

train = train[~(((train.site_id==5) &(train.cnt_tmp_meter_siteid/train.cnt_building_per_site>.35) & (train.meter_reading==0)))]
print(train.shape)

train = train[~(((train.site_id==6) &(train.cnt_tmp_meter_siteid/train.cnt_building_per_site>.35) & (train.meter_reading==0)))]
print(train.shape)

train = train[~(((train.site_id==8) &(train.cnt_tmp_meter_siteid/train.cnt_building_per_site>.35) & (train.meter_reading==0)))]
print(train.shape)

train = train[~(((train.site_id==9) &(train.cnt_tmp_meter_siteid/train.cnt_building_per_site>.35) & (train.meter_reading==0)))]
print(train.shape)

train = train[~(((train.site_id==14) &(train.cnt_tmp_meter_siteid/train.cnt_building_per_site>.35) & (train.meter_reading==0)))]
print(train.shape)

train = train[~(((train.site_id==15) &(train.cnt_tmp_meter_siteid/train.cnt_building_per_site>.35) & (train.meter_reading==0)))]
print(train.shape)

train = train[['building_id', 'meter', 'timestamp', 'meter_reading']].reset_index(drop=True)

train.to_feather('train_cleanup_001.feather')
# -



# # 4. clean up datasets (train & test) 
#
# ####### code extracted from https://www.kaggle.com/purist1024/ashrae-simple-data-cleanup-lb-1-08-no-leaks

# +
def make_is_bad_zero(Xy_subset, min_interval=48, summer_start=3000, summer_end=7500):
    """Helper routine for 'find_bad_zeros'.
    
    This operates upon a single dataframe produced by 'groupby'. We expect an 
    additional column 'meter_id' which is a duplicate of 'meter' because groupby 
    eliminates the original one."""
    meter = Xy_subset.meter_id.iloc[0]
    is_zero = Xy_subset.meter_reading == 0
    if meter == 0:
        # Electrical meters should never be zero. Keep all zero-readings in this table so that
        # they will all be dropped in the train set.
        return is_zero

    transitions = (is_zero != is_zero.shift(1))
    all_sequence_ids = transitions.cumsum()
    ids = all_sequence_ids[is_zero].rename("ids")
    if meter in [2, 3]:
        # It's normal for steam and hotwater to be turned off during the summer
        keep = set(ids[(Xy_subset.timestamp < summer_start) |
                       (Xy_subset.timestamp > summer_end)].unique())
        is_bad = ids.isin(keep) & (ids.map(ids.value_counts()) >= min_interval)
    elif meter == 1:
        time_ids = ids.to_frame().join(Xy_subset.timestamp).set_index("timestamp").ids
        is_bad = ids.map(ids.value_counts()) >= min_interval

        # Cold water may be turned off during the winter
        jan_id = time_ids.get(0, False)
        dec_id = time_ids.get(8283, False)
        if (jan_id and dec_id and jan_id == time_ids.get(500, False) and
                dec_id == time_ids.get(8783, False)):
            is_bad = is_bad & (~(ids.isin(set([jan_id, dec_id]))))
    else:
        raise Exception(f"Unexpected meter type: {meter}")

    result = is_zero.copy()
    result.update(is_bad)
    return result

def find_bad_zeros(X, y):
    """Returns an Index object containing only the rows which should be deleted."""
    Xy = X.assign(meter_reading=y, meter_id=X.meter)
    is_bad_zero = Xy.groupby(["building_id", "meter"]).apply(make_is_bad_zero)
    return is_bad_zero[is_bad_zero].index.droplevel([0, 1])


# -

# `find_bad_sitezero` identifies the "known-bad" electrical readings from the first 141 days of the data for site 0 (i.e. UCF).

def find_bad_sitezero(X):
    """Returns indices of bad rows from the early days of Site 0 (UCF)."""
    return X[(X.timestamp < 3378) & (X.site_id == 0) & (X.meter == 0)].index


# `find_bad_building1099` identifies the most absurdly high readings from building 1099. These are orders of magnitude higher than all data, and have been emperically seen in LB probes to be harmful outliers.

def find_bad_building1099(X, y):
    """Returns indices of bad rows (with absurdly high readings) from building 1099."""
    return X[(X.building_id == 1099) & (X.meter == 2) & (y > 3e4)].index


# Finally, `find_bad_rows` combines all of the above together to allow you to do a one-line cleanup of your data.

def find_bad_rows(X, y):
    return find_bad_zeros(X, y).union(find_bad_sitezero(X)).union(find_bad_building1099(X, y))


# +
from meteocalc import Temp, dew_point, heat_index, wind_chill, feels_like

def c2f(T):
    return T * 9 / 5. + 32

def windchill(T, v):
    return (10*v**.5 - v +10.5) * (33 - T)

def prepareweather(df):
    df['RH'] = 100 - 5 * (df['air_temperature']-df['dew_temperature'])
#     df['RH_above50'] = (df['RH'] > 50).astype(int) 
    df['heat'] = df.apply(lambda x: heat_index(c2f(x.air_temperature), x.RH).c, axis=1)
    df['windchill'] = df.apply(lambda x: windchill(x.air_temperature, x.wind_speed), axis=1)
    df['feellike'] = df.apply(lambda x: feels_like(c2f(x.air_temperature), x.RH, x.wind_speed*2.237).c, axis=1)
    
    return df



def add_lag_feature(weather_df, window=3):  
    group_df = weather_df.groupby('site_id')
    cols = ['air_temperature', 'dew_temperature', 'heat', 'windchill', 'feellike']
    rolled = group_df[cols].rolling(window=window, min_periods=0)
    lag_mean = rolled.mean().reset_index().astype(np.float32)
    lag_max = rolled.max().reset_index().astype(np.float16)
    lag_min = rolled.min().reset_index().astype(np.float16)
    lag_std = rolled.std().reset_index().astype(np.float16)
    for col in cols:
        weather_df[f'{col}_mean_lag{window}'] = lag_mean[col]
#         weather_df[f'{col}_max_lag{window}'] = lag_max[col]
#         weather_df[f'{col}_min_lag{window}'] = lag_min[col]
#         weather_df[f'{col}_std_lag{window}'] = lag_std[col]


def fill_weather_dataset(weather_df):
    
    
    
    # Find Missing Dates
    time_format = "%Y-%m-%d %H:%M:%S"
    start_date = datetime.datetime.strptime(weather_df['timestamp'].min(),time_format)
    end_date = datetime.datetime.strptime(weather_df['timestamp'].max(),time_format)
    total_hours = int(((end_date - start_date).total_seconds() + 3600) / 3600)
    hours_list = [(end_date - datetime.timedelta(hours=x)).strftime(time_format) for x in range(total_hours)]

    missing_hours = []
    for site_id in range(16):
        site_hours = np.array(weather_df[weather_df['site_id'] == site_id]['timestamp'])
        new_rows = pd.DataFrame(np.setdiff1d(hours_list,site_hours),columns=['timestamp'])
        new_rows['site_id'] = site_id
        weather_df = pd.concat([weather_df,new_rows])

        weather_df = weather_df.reset_index(drop=True) 
        
#     for col in weather_df.columns:
#         if col != 'timestamp':
#             if weather_df[col].isna().sum():
#                 weather_df['na_'+col] = weather_df[col].isna().astype(int)

#     weather_df['weath_na_total'] = weather_df.isna().sum(axis=1)
    
    
    # Add new Features
    weather_df["datetime"] = pd.to_datetime(weather_df["timestamp"])
    weather_df["day"] = weather_df["datetime"].dt.day
    weather_df["week"] = weather_df["datetime"].dt.week
    weather_df["month"] = weather_df["datetime"].dt.month
    
    # Reset Index for Fast Update
    weather_df = weather_df.set_index(['site_id','day','month'])

    air_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['air_temperature'].mean(),columns=["air_temperature"])
    weather_df.update(air_temperature_filler,overwrite=False)

    # Step 1
    cloud_coverage_filler = weather_df.groupby(['site_id','day','month'])['cloud_coverage'].mean()
    # Step 2
    cloud_coverage_filler = pd.DataFrame(cloud_coverage_filler.fillna(method='ffill'),columns=["cloud_coverage"])

    weather_df.update(cloud_coverage_filler,overwrite=False)

    due_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['dew_temperature'].mean(),columns=["dew_temperature"])
    weather_df.update(due_temperature_filler,overwrite=False)

    # Step 1
    sea_level_filler = weather_df.groupby(['site_id','day','month'])['sea_level_pressure'].mean()
    # Step 2
    sea_level_filler = pd.DataFrame(sea_level_filler.fillna(method='ffill'),columns=['sea_level_pressure'])

    weather_df.update(sea_level_filler,overwrite=False)

    wind_direction_filler =  pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_direction'].mean(),columns=['wind_direction'])
    weather_df.update(wind_direction_filler,overwrite=False)

    wind_speed_filler =  pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_speed'].mean(),columns=['wind_speed'])
    weather_df.update(wind_speed_filler,overwrite=False)

    # Step 1
    precip_depth_filler = weather_df.groupby(['site_id','day','month'])['precip_depth_1_hr'].mean()
    # Step 2
    precip_depth_filler = pd.DataFrame(precip_depth_filler.fillna(method='ffill'),columns=['precip_depth_1_hr'])

    weather_df.update(precip_depth_filler,overwrite=False)

    weather_df = weather_df.reset_index()
    weather_df = weather_df.drop(['datetime','day','week','month'],axis=1)
    
    
    
    weather_df = timestamp_align(weather_df)
    
    print('add heat, RH...')
    weather_df = prepareweather(weather_df)
    
    print('add lag features')
    add_lag_feature(weather_df, window=3)
    
    return weather_df




# Original code from https://www.kaggle.com/gemartin/load-data-reduce-memory-usage by @gemartin

from pandas.api.types import is_datetime64_any_dtype as is_datetime
from pandas.api.types import is_categorical_dtype

def reduce_mem_usage(df, use_float16=False):
    """
    Iterate through all the columns of a dataframe and modify the data type to reduce memory usage.        
    """
    
    start_mem = df.memory_usage().sum() / 1024**2
    print("Memory usage of dataframe is {:.2f} MB".format(start_mem))
    
    for col in df.columns:
        if is_datetime(df[col]) or is_categorical_dtype(df[col]):
            continue
        col_type = df[col].dtype
        
        if col_type != object:
            c_min = df[col].min()
            c_max = df[col].max()
            if str(col_type)[:3] == "int":
                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
                    df[col] = df[col].astype(np.int8)
                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
                    df[col] = df[col].astype(np.int16)
                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
                    df[col] = df[col].astype(np.int32)
                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
                    df[col] = df[col].astype(np.int64)  
            else:
                if use_float16 and c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
                    df[col] = df[col].astype(np.float16)
                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
                    df[col] = df[col].astype(np.float32)
                else:
                    df[col] = df[col].astype(np.float64)
        else:
            df[col] = df[col].astype("category")

    end_mem = df.memory_usage().sum() / 1024**2
    print("Memory usage after optimization is: {:.2f} MB".format(end_mem))
    print("Decreased by {:.1f}%".format(100 * (start_mem - end_mem) / start_mem))
    
    return df


def features_engineering(df):
    
    # Sort by timestamp
    df.sort_values("timestamp")
    df.reset_index(drop=True)
    
    # Add more features
    df["timestamp"] = pd.to_datetime(df["timestamp"],format="%Y-%m-%d %H:%M:%S")
    df["hour"] = df["timestamp"].dt.hour
    df["month"] = df["timestamp"].dt.month
    df["weekday"] = df["timestamp"].dt.weekday

    
#     # Remove Unused Columns
#     drop = ['sea_level_pressure", "wind_direction", "wind_speed", "precip_depth_1_hr"]
#     df = df.drop(drop, axis=1)
    gc.collect()
    
    return df


def building_features(building_meta_df):
    building_addfeatures = pd.read_feather('building_all_meters.feather')
    
    for col in building_meta_df.columns:
        if col != 'timestamp':
            if building_meta_df[col].isna().sum():
                building_meta_df['na_'+col] = building_meta_df[col].isna().astype(int)

    building_meta_df['build_na_total'] = building_meta_df.isna().sum(axis=1)

    building_meta_df = pd.concat([building_meta_df, 
                                  building_addfeatures[['meter_reading_0', 'meter_reading_1', 
                                                        'meter_reading_2', 'meter_reading_3']]], axis=1)

    from sklearn.preprocessing import LabelEncoder
    le = LabelEncoder()
    building_meta_df.primary_use = le.fit_transform(building_meta_df.primary_use)

    building_meta_df['cnt_building_per_site'] = building_meta_df.groupby(['site_id']).building_id.transform(lambda x: x.size)
    building_meta_df['cnt_building_per_site_prim'] = building_meta_df.groupby(['site_id', 'primary_use']).building_id.transform(lambda x: x.size)
    building_meta_df['sqr_mean_per_site'] = building_meta_df.groupby(['site_id', ]).square_feet.transform('median')
    building_meta_df['sqr_mean_per_prim_site'] = building_meta_df.groupby(['site_id', 'primary_use']).square_feet.transform('median')

    return building_meta_df


# +
import pandas as pd
import numpy as np
import os, gc
import warnings

from lightgbm import LGBMRegressor
from sklearn.base import BaseEstimator, RegressorMixin, clone
from sklearn.metrics import mean_squared_log_error

pd.set_option("max_columns", 500)

def input_file(file):
    path = f"./{file}"
    if not os.path.exists(path): return path + ".gz"
    return path

def compress_dataframe(df):
    result = df.copy()
    for col in result.columns:
        col_data = result[col]
        dn = col_data.dtype.name
        if dn == "object":
            result[col] = pd.to_numeric(col_data.astype("category").cat.codes, downcast="integer")
        elif dn == "bool":
            result[col] = col_data.astype("int8")
        elif dn.startswith("int") or (col_data.round() == col_data).all():
            result[col] = pd.to_numeric(col_data, downcast="integer")
        else:
            result[col] = pd.to_numeric(col_data, downcast='float')
    return result

def read_train():
    df = pd.read_feather("./train_cleanup_001.feather")
    dft = pd.read_csv('test.csv')
    
#     df = features_engineering(df) ###############################
    df.timestamp = (pd.to_datetime(df["timestamp"]) - pd.to_datetime("2016-01-01")).dt.total_seconds() // 3600
    return compress_dataframe(df)

def read_building_metadata():
     
    df = pd.read_csv(input_file("building_metadata.csv"))
    df = building_features(df)                 
    df = compress_dataframe(df).fillna(-1).set_index("building_id")
    return df

site_GMT_offsets = [-5, 0, -7, -5, -8, 0, -5, -5, -5, -6, -7, -5, 0, -6, -5, -5]

def read_weather_train(fix_timestamps=True, interpolate_na=True, add_na_indicators=True):
    df = pd.read_csv(input_file("weather_train.csv"), parse_dates=["timestamp"])
            ################
    
    print('add heat, RH...')
    df = prepareweather(df)
    
    print('add lag features')
    add_lag_feature(df, window=3)
            #################
    
    df.timestamp = (df.timestamp - pd.to_datetime("2016-01-01")).dt.total_seconds() // 3600
    if fix_timestamps:
        GMT_offset_map = {site: offset for site, offset in enumerate(site_GMT_offsets)}
        df.timestamp = df.timestamp + df.site_id.map(GMT_offset_map)
    if interpolate_na:
        site_dfs = []
        for site_id in df.site_id.unique():
            # Make sure that we include all possible hours so that we can interpolate evenly
            site_df = df[df.site_id == site_id].set_index("timestamp").reindex(range(8784))
            site_df.site_id = site_id
            for col in [c for c in site_df.columns if c != "site_id"]:
                if add_na_indicators: site_df[f"had_{col}"] = ~site_df[col].isna()
                site_df[col] = site_df[col].interpolate(limit_direction='both', method='linear')
                # Some sites are completely missing some columns, so use this fallback
                site_df[col] = site_df[col].fillna(df[col].median())
            site_dfs.append(site_df)
        df = pd.concat(site_dfs).reset_index()  # make timestamp back into a regular column
    elif add_na_indicators:
        for col in df.columns:
            if df[col].isna().any(): df[f"had_{col}"] = ~df[col].isna()
    return compress_dataframe(df).set_index(["site_id", "timestamp"])

def combined_train_data(fix_timestamps=True, interpolate_na=True, add_na_indicators=True):
    Xy = compress_dataframe(read_train().join(read_building_metadata(), on="building_id").join(
        read_weather_train(fix_timestamps, interpolate_na, add_na_indicators),
        on=["site_id", "timestamp"]).fillna(-1))
    return Xy.drop(columns=["meter_reading"]), Xy.meter_reading

def _add_time_features(X):
    return X.assign(tm_day_of_week=((X.timestamp // 24) % 7), tm_hour_of_day=(X.timestamp % 24))



# +
def read_test():
    df = pd.read_csv(input_file("test.csv"), parse_dates=["timestamp"])
    df.timestamp = (df.timestamp - pd.to_datetime("2016-01-01")).dt.total_seconds() // 3600
    return compress_dataframe(df).set_index("row_id")

def read_weather_test(fix_timestamps=True, interpolate_na=True, add_na_indicators=True):
    df = pd.read_csv(input_file("weather_test.csv"), parse_dates=["timestamp"])
    
    ###############
    print('add heat, RH...')
    df = prepareweather(df)
    
    print('add lag features')
    add_lag_feature(df, window=3)
    ##############
    
    
    
    
    
    df.timestamp = (df.timestamp - pd.to_datetime("2016-01-01")).dt.total_seconds() // 3600
    if fix_timestamps:
        GMT_offset_map = {site: offset for site, offset in enumerate(site_GMT_offsets)}
        df.timestamp = df.timestamp + df.site_id.map(GMT_offset_map)
    if interpolate_na:
        site_dfs = []
        for site_id in df.site_id.unique():
            # Make sure that we include all possible hours so that we can interpolate evenly
            site_df = df[df.site_id == site_id].set_index("timestamp").reindex(range(8784, 26304))
            site_df.site_id = site_id
            for col in [c for c in site_df.columns if c != "site_id"]:
                if add_na_indicators: site_df[f"had_{col}"] = ~site_df[col].isna()
                site_df[col] = site_df[col].interpolate(limit_direction='both', method='linear')
                # Some sites are completely missing some columns, so use this fallback
                site_df[col] = site_df[col].fillna(df[col].median())
            site_dfs.append(site_df)
        df = pd.concat(site_dfs).reset_index()  # make timestamp back into a regular column
    elif add_na_indicators:
        for col in df.columns:
            if df[col].isna().any(): df[f"had_{col}"] = ~df[col].isna()
    return compress_dataframe(df).set_index(["site_id", "timestamp"])

def combined_test_data(fix_timestamps=True, interpolate_na=True, add_na_indicators=True):
    X = compress_dataframe(read_test().join(read_building_metadata(), on="building_id").join(
        read_weather_test(fix_timestamps, interpolate_na, add_na_indicators),
        on=["site_id", "timestamp"]).fillna(-1))
    return X


# +
X, y = combined_train_data()

bad_rows = find_bad_rows(X, y)
pd.Series(bad_rows.sort_values()).to_csv("rows_to_drop.csv", header=False, index=False)
# -

categorical_columns = [
    "building_id", "meter", "site_id", "primary_use", "had_air_temperature", "had_cloud_coverage","had_dew_temperature", "had_precip_depth_1_hr", "had_sea_level_pressure", "had_wind_direction","had_wind_speed", "tm_day_of_week", "tm_hour_of_day"
]

# +
dropcol = []

categorical_columns = [f for f in categorical_columns if f not in dropcol]
# -

X.columns

# +
X = X.drop(index=bad_rows)
y = y.reindex_like(X)

# Additional preprocessing
X = compress_dataframe(_add_time_features(X))

X = X.drop(dropcol, axis=1)  # Raw timestamp doesn't help when prediction
y = np.log1p(y)
# -

y

XX = X.copy()
XX['meter_reading'] = y.values
XX.reset_index(drop=True, inplace=True)
XX.to_feather('train_simple_cleanup.feather')

Xt = combined_test_data()
Xt = compress_dataframe(_add_time_features(Xt))
Xt = Xt.drop(dropcol, axis=1)  # Raw timestamp doesn't help when prediction
Xt = Xt.reset_index()
Xt.to_feather('test_simple_cleanup.feather')



StarcoderdataJupyterScriptsDedupFiltered