<jupyter_start><jupyter_text>Examine price<jupyter_code>sns.histplot(df.price, bins=100)
sns.histplot(np.log1p(df.price), bins=50)<jupyter_output><empty_output><jupyter_text>Data Wrangling<jupyter_code>columns_of_interest = [ 'latitude'
                      , 'longitude'
                      , 'price'
                      , 'minimum_nights'
                      , 'number_of_reviews'
                      , 'reviews_per_month'
                      , 'calculated_host_listings_count'
                      ,'availability_365']
df = df[columns_of_interest]
df.head()<jupyter_output><empty_output><jupyter_text>Q1: Missing Values<jupyter_code>df.isnull().sum()<jupyter_output><empty_output><jupyter_text>Q2: Median minimum_nights<jupyter_code>df.minimum_nights.median()<jupyter_output><empty_output><jupyter_text>Split data<jupyter_code>n = len(df)
n_val = int(n * 0.2)
n_test = int(n * 0.2)
n_train = n - n_val - n_test

idx = np.arange(n)

np.random.seed(42)
np.random.shuffle(idx)

df_train      = df.iloc[idx[:n_train]]              .reset_index(drop=True)
df_test       = df.iloc[idx[n_train:n_train+n_test]].reset_index(drop=True)
df_val        = df.iloc[idx[n_train+n_test:]]       .reset_index(drop=True)
df_train_full = df.iloc[idx[:n_train+n_test]]       .reset_index(drop=True)

y_train      = np.log1p(df_train.price)
y_test       = np.log1p(df_test.price)
y_val        = np.log1p(df_val.price)
y_train_full = np.log1p(df_train_full.price)

del df_train     ['price']
del df_test      ['price']
del df_val       ['price']
del df_train_full['price']<jupyter_output><empty_output><jupyter_text>TRAINING FUNCTION<jupyter_code>def prepare_X_zero(df):
    df = df.copy()
    df['reviews_per_month'] = df.reviews_per_month.fillna(0)
    X = df.values
    return X

def prepare_X_mean(df):
    df = df.copy()
    df['reviews_per_month'] = df.reviews_per_month.fillna(df.reviews_per_month.mean())
    X = df.values
    return X

def train_linear_regression_reg(X, y, r=0.001):
    X = np.column_stack([np.ones(X.shape[0]), X])
    XTX = X.T.dot(X)
    XTX = XTX + r * np.eye(XTX.shape[0])
    XTX_inv = np.linalg.inv(XTX)
    
    w_full = XTX_inv.dot(X.T).dot(y)
    
    return w_full[0], w_full[1:]

def rmse(y, y_pred):
    se = (y - y_pred) ** 2
    mse = se.mean()
    return np.sqrt(mse)

X_train_zero = prepare_X_zero(df_train)
X_val_zero   = prepare_X_zero(df_val)
X_train_mean = prepare_X_mean(df_train)
X_val_mean   = prepare_X_mean(df_val)

w0_zero, w_zero = train_linear_regression_reg(X_train_zero,y_train,r=0)
w0_mean, w_mean = train_linear_regression_reg(X_train_mean,y_train,r=0)

y_pred_zero = w0_zero + X_val_zero.dot(w_zero)
y_pred_mean = w0_mean + X_val_zero.dot(w_mean)

print('reviews_per_month na -> 0   : {}'.format(round(rmse(y_val,y_pred_zero),2)))
print('reviews_per_month na -> mean: {}'.format(round(rmse(y_val,y_pred_mean),2)))<jupyter_output>reviews_per_month na -> 0   : 0.65
reviews_per_month na -> mean: 0.65<jupyter_text>Neither is better to this accuracy, thechnically 0 is slightly better. But for the next question we use 0.  Q4: What regularisation to use<jupyter_code>prepare_X = prepare_X_zero
X_train = prepare_X(df_train)
X_val   = prepare_X(df_val)

res = []
for r in [0, 0.000001, 0.0001, 0.001, 0.01, 0.1, 1, 5, 10]:
    w0, w = train_linear_regression_reg(X_train,y_train,r=r)
    y_pred = w0 + X_val.dot(w)
    error = rmse(y_val,y_pred)
    print('{:10}:{} ({})'.format(r,error,round(error,2)))<jupyter_output>0:0.6536337649996121 (0.65)
     1e-06:0.6536338933922969 (0.65)
    0.0001:0.653651793240982 (0.65)
     0.001:0.6541885180213967 (0.65)
      0.01:0.6656175809489505 (0.67)
       0.1:0.6869575817254718 (0.69)
         1:0.6919185299146808 (0.69)
         5:0.6923850940178745 (0.69)
        10:0.6924224370107044 (0.69)<jupyter_text>smallest error when r=0<jupyter_code>r = 0<jupyter_output><empty_output><jupyter_text>Q5 What effect does seed have<jupyter_code>def split_by_seed(seed):
    idx = np.arange(n)

    np.random.seed(seed)
    np.random.shuffle(idx)

    df_train      = df.iloc[idx[:n_train]]              .reset_index(drop=True)
    df_test       = df.iloc[idx[n_train:n_train+n_test]].reset_index(drop=True)
    df_val        = df.iloc[idx[n_train+n_test:]]       .reset_index(drop=True)
    df_train_full = df.iloc[idx[:n_train+n_test]]       .reset_index(drop=True)

    y_train      = np.log1p(df_train.price)
    y_test       = np.log1p(df_test.price)
    y_val        = np.log1p(df_val.price)
    y_train_full = np.log1p(df_train_full.price)

    del df_train     ['price']
    del df_test      ['price']
    del df_val       ['price']
    del df_train_full['price']
    return df_train, df_test, df_val, df_train_full, y_train, y_test, y_val, y_train_full

def calculate_rmse_by_seed(seed):
    df_train, df_test, df_val, df_train_full, y_train, y_test, y_val, y_train_full = split_by_seed(seed)
    
    X_train = prepare_X(df_train)
    X_val   = prepare_X(df_val)
    w0, w = train_linear_regression_reg(X_train,y_train,r=r)
    y_pred = w0 + X_val.dot(w)
    error = rmse(y_val,y_pred)
    return error

seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
errors = [ calculate_rmse_by_seed(seed) for seed in seeds]
print(errors)
print(np.std(errors))
print(np.round(np.std(errors),3))<jupyter_output>[0.6304569982323267, 0.6486723821210312, 0.6409503261201178, 0.6396698468034724, 0.6555733448818004, 0.6391858794518651, 0.6473543640304936, 0.655889218349002, 0.6405135568815358, 0.6450366766956963]
0.007450888539027653
0.007<jupyter_text>Q6: Final model error<jupyter_code>seed = 42
df_train, df_test, df_val, df_train_full, y_train, y_test, y_val, y_train_full = split_by_seed(seed)
X_train_full = prepare_X(df_train_full)
X_test       = prepare_X(df_test)

w0, w = train_linear_regression_reg(X_train_full,y_train_full,r=0.001)
y_test_pred = w0 + X_test.dot(w)
rmse(y_test,y_test_pred)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup