# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
from sklearn.manifold import TSNE
import random
from matplotlib import pyplot
sns.set()
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn import preprocessing
from sklearn.preprocessing import FunctionTransformer
from keras.layers import Dense, Dropout
from keras.models import Sequential
pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)
pd.set_option('display.width', 2000)
from keras.optimizers import SGD
from sklearn.impute import SimpleImputer
from sklearn.pipeline import make_pipeline, Pipeline, FeatureUnion

import keras
from tensorflow.python import debug as tf_debug
from tensorflow.python.client import device_lib
#print(device_lib.list_local_devices())
import import_ipynb
from Helpers import *
from PandasFeatureUnion import PandasFeatureUnion

# +
train = pd.read_csv('train.csv', index_col='Id')
test = pd.read_csv('test.csv',index_col='Id')
test_ids = test.index
y_train = np.log(train['SalePrice'])
np.random.seed(1)
random.seed(1)
total_bath= lambda x: x.FullBath+x.HalfBath*0.5 +x.BsmtHalfBath*0.5+x.BsmtFullBath


bath_columns = ['FullBath','HalfBath','BsmtHalfBath', 'BsmtFullBath']
bath_pipeline = Pipeline([
    ('SelectBathColumns',  ColumnSelector(columns=bath_columns)),
    ('fillNaAsZero', FillNA(columns=bath_columns, fill_value=0)),
    ('combine', ApplyTransformer(fn=total_bath, name='Bath')),
  #  ('drop',DropColumns(columns=bath_columns))
]) 
  

important_columns = [
    'GrLivArea', 'MasVnrArea', 'LotArea',  '1stFlrSF',
    
    'YearBuilt','YearRemodAdd',
    
    'OverallQual','MSSubClass',
    'Neighborhood', 'SaleType', 'SaleCondition',
    'BldgType','LotFrontage','GarageCars','CentralAir',
    'PoolQC', 'BsmtQual','ExterQual', 'KitchenQual',
    'MasVnrType', 'HouseStyle'
]
# No improovement
# 'BldgType' 

cs = ColumnSelector(columns=important_columns)

scdt = DummiesTransformer(column='MSSubClass', keys=[ 20,  30,  40,  45,  50,  60,  70,  75,  80,  85,  90, 120, 150, 160, 180, 190])
oqdt = DummiesTransformer(column='OverallQual', keys=range(1,11))
neigbTrans = DummiesTransformer(column='Neighborhood', keys=["Blmngtn", "Blueste", "BrDale", "BrkSide", "ClearCr", "CollgCr", "Crawfor", "Edwards", "Gilbert", "IDOTRR", "MeadowV", "Mitchel", "Names", "NoRidge", "NPkVill", "NridgHt", "NWAmes", "OldTown", "SWISU", "Sawyer", "SawyerW", "Somerst", "StoneBr", "Timber", "Veenker"])
bdTypeTrans= DummiesTransformer(column='BldgType', keys=['1Fam' ,'TwnhsE', 'Twnhs', 'Duplex', '2fmCon'])
saleTypeTrans= DummiesTransformer(column='SaleType', keys=["WD","CWD","VWD","New","COD","Con","ConLw","ConLI","ConLD","Oth"])
bsmtQualTrans = DummiesTransformer(column='BsmtQual', keys=['Ex','Gd','TA','Fa','Po','None'])
extQualTrans = DummiesTransformer(column='ExterQual', keys=['Ex','Gd','TA','Fa','Po','None'])
kitQualTrans = DummiesTransformer(column='KitchenQual', keys=['Ex','Gd','TA','Fa','Po','None'])
saleCondTrans = DummiesTransformer(column='SaleCondition', keys=['Normal', 'Abnorml',
                                                                'AdjLand','Alloca', 
                                                                 'Family', 'Partial'])
masVnrTypeTrans = DummiesTransformer(column='MasVnrType', keys=['BrkCmn', 'BrkFace', 
                                                                'CBlock','None', 'Stone'])
lotConfigTrans = DummiesTransformer(column='LotConfig', keys=['Inside', 'Corner', 'CulDSac',
                                                             'FR2', 'FR3'])
aircTrans = DummiesTransformer(column='CentralAir', keys=['Y', 'N'])
houseStyleTrans = DummiesTransformer(column='HouseStyle', keys=['1Story','1.5Fin','1.5Unf',
                                                               '2Story','2.5Fin','2.5Unf',
                                                               'SFoyer','SLvl'])

core_pipeline = Pipeline(
    [('SelectColumns', cs),
     ('DummiesMSSubClass', scdt),
     ('OverallQual', oqdt),
     ('DummiesNeighborhood', neigbTrans),
     ('DummiesSaleType', saleTypeTrans),
     ('DummiesBldgType',bdTypeTrans),
     ('fillNaAsZero', FillNA(columns=['LotFrontage','GarageCars','MasVnrArea'], fill_value=0)),
     ('fillNaAsNone', FillNA(columns=['BsmtQual','ExterQual','KitchenQual', 
                                     'SaleCondition','MasVnrType'], fill_value='None')),
     ('LogLotFrontage', ApplyTransformer(fn=lambda x:np.log1p(x.LotFrontage), name='LotFrontage')),
     ('LogGrLivArea', ApplyTransformer(fn=lambda x:np.log1p(x.GrLivArea), name='GrLivArea')),
     ('LogMasVnrArea', ApplyTransformer(fn=lambda x:np.log1p(x.GrLivArea), name='MasVnrArea')),
     ('Log1stFlrSF', ApplyTransformer(fn=lambda x:np.log1p(x['1stFlrSF']), name='1stFlrSF')),
     ('PoolQC_Exists', ApplyTransformer(fn=lambda x:0 if (pd.isnull(x['PoolQC'])) else 1, name='PoolQC')),
     ('BsmtQualTrans', bsmtQualTrans),
     ('ExtQualTrans', extQualTrans),
     ('kitQualTrans', kitQualTrans),
     ('saleCondTrans', saleCondTrans),
      ('houseStyletrans', houseStyleTrans),
     ('masVnrTypeTrans',masVnrTypeTrans),
     ('aircTrans', aircTrans)
    ])

pp = PandasFeatureUnion([
    ('bath',bath_pipeline),
    ('core', core_pipeline)], n_jobs=None)

X_train = pp.fit_transform(train)
X_test = pp.transform(test)
 
 

# +
# define model
# cc=keras.constraints.MinMaxNorm(min_value=0.0, max_value=2.0, rate=1, axis=0)
model = Sequential()
input_layer = Dense(20, 
#                     kernel_constraint=cc,
#                     kernel_regularizer=keras.regularizers.l1_l2(l1=0.01, l2=0.01),
#                     bias_regularizer=keras.regularizers.l1_l2(l1=0.01, l2=0.01),
                    input_dim=len(X_train.columns), 
                    activation='relu', 
                    kernel_initializer='normal') 
model.add(input_layer)
model.add(Dropout(rate=0.1))

model.add(Dense(50, activation='relu', kernel_initializer='normal') )
model.add(Dense(25, activation='relu', kernel_initializer='normal') )
model.add(Dense(10, activation='relu', kernel_initializer='normal') )
model.add(Dense(1, activation='linear'))

optimizer = keras.optimizers.SGD(lr=0.001) #Adam(lr=0.0015);
# compile model
model.compile(loss='mean_squared_error', optimizer=optimizer)

# +

trainX, testX, trainy, testy = train_test_split(X_train, 
                                                    y_train, 
                                                    test_size=0.3, 
                                                    random_state=1)

trainX_ids = trainX.index
scaler = preprocessing.MinMaxScaler(feature_range=(0, 1))
scaler.fit(trainX)
trainX= scaler.transform(trainX)
testX= scaler.transform(testX)
testData = X_test
X_test=scaler.transform(X_test)
# fit model
early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0.0001, 
                                           patience=100, verbose=1, mode='auto', 
                                           baseline=None, restore_best_weights=True)
tb_callback =keras.callbacks.TensorBoard(log_dir='./logs', histogram_freq=0, 
                                         batch_size=32, write_graph=True, 
                                         write_grads=True, write_images=True,
                                         embeddings_freq=0, embeddings_layer_names=None,
                                         embeddings_metadata=None, embeddings_data=None,
                                         update_freq='epoch')
history = model.fit(trainX, trainy, validation_data=(testX, testy), 
                    epochs=4000, verbose=0, batch_size=32, callbacks=[early_stop, tb_callback])

drawTFModel(history, model, trainX, trainy, testX, testy)
# -

result = model.predict(X_test)
dfR = pd.DataFrame(test_ids,columns=['Id'])
dfR['SalePrice']=np.exp(result)
dfR.to_csv('predictions.csv',index = None, header=True)

# +
tt = pd.DataFrame(testy.copy())
trainedPrice = model.predict(testX)
tt['PredictedLog'] = trainedPrice
tt['Predicted']=np.exp(trainedPrice)
tt['Real']=np.exp(testy)
tt['Delta'] = tt.Predicted - tt.Real
tt['DeltaLog'] =tt.PredictedLog - tt.SalePrice
plt.scatter(tt.index, tt.Delta)
plt.show()
plt.scatter(tt.index, tt.DeltaLog)
 
plt.show()


problematicIDs = tt[np.abs(tt.DeltaLog) >0.2]


train.loc[problematicIDs.index, :]

# -

tt.describe()

df=train

# +
# #!rm -rf logs
# -

pd.DataFrame(X_test, columns=testData.columns).describe().T

test_ids

testData.index

testData.loc[2577,:]

n=lambda x: (1 if (pd.isnull(x)) else 0)

n(np.nan)

n(55)


# +
# Utility function to visualize the outputs of PCA and t-SNE

def fashion_scatter(x, colors):
    # choose a color palette with seaborn.
    num_classes = len(np.unique(colors))
    palette = np.array(sns.color_palette("hls", num_classes))

    # create a scatter plot.
    f = plt.figure(figsize=(8, 8))
    ax = plt.subplot(aspect='equal')
    sc = ax.scatter(x[:,0], x[:,1], lw=0, s=40, c=palette[colors] )
    ax.grid(linestyle='-', linewidth='0.5', color='red')
    plt.xlim(-25, 25)
    plt.ylim(-25, 25)
    ax.axis('off')
    ax.axis('tight')

    # add the labels for each digit corresponding to the label
    txts = []

    for i in range(num_classes):

        # Position of each label at median of data points.

        xtext, ytext = np.median(x[colors == i, :], axis=0)
        txt = ax.text(xtext, ytext, str(i), fontsize=24)
        txt.set_path_effects([
            PathEffects.Stroke(linewidth=5, foreground="w"),
            PathEffects.Normal()])
        txts.append(txt)

    return f, ax, sc, txts


# -

n(None)

pd.isnull('sss')

d1 = X_train.copy()
d1['SalePrice'] = y_train

import time
time_start = time.time()
tsne = TSNE(random_state=1, n_components=2, perplexity=2500).fit_transform(d1)
print( 't-SNE done! Time elapsed: {} seconds'.format(time.time()-time_start))

 fashion_scatter(tsne, pd.Series([0]))



tsne[0]

len(tsne)



StarcoderdataJupyterScriptsDedupFiltered