# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] colab_type="text" id="view-in-github"
# <a href="https://colab.research.google.com/github/mpHarm88/DS-Unit-2-Regression-Classification/blob/master/module3/Mikio_Harman_assignment_regression_classification_3.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + [markdown] colab_type="text" id="7IXUfiQ2UKj6"
# Lambda School Data Science, Unit 2: Predictive Modeling
#
# # Regression & Classification, Module 3
#
# ## Assignment
#
# We're going back to our other **New York City** real estate dataset. Instead of predicting apartment rents, you'll predict property sales prices.
#
# But not just for condos in Tribeca...
#
# Instead, predict property sales prices for **One Family Dwellings** (`BUILDING_CLASS_CATEGORY` == `'01 ONE FAMILY DWELLINGS'`). 
#
# Use a subset of the data where the **sale price was more than \\$100 thousand and less than $2 million.** 
#
# The [NYC Department of Finance](https://www1.nyc.gov/site/finance/taxes/property-rolling-sales-data.page) has a glossary of property sales terms and NYC Building Class Code Descriptions. The data comes from the [NYC OpenData](https://data.cityofnewyork.us/browse?q=NYC%20calendar%20sales) portal.
#
# - [ ] Do train/test split. Use data from January — March 2019 to train. Use data from April 2019 to test.
# - [ ] Do one-hot encoding of categorical features.
# - [ ] Do feature selection with `SelectKBest`.
# - [ ] Do [feature scaling](https://scikit-learn.org/stable/modules/preprocessing.html).
# - [ ] Fit a ridge regression model with multiple features.
# - [ ] Get mean absolute error for the test set.
# - [ ] As always, commit your notebook to your fork of the GitHub repo.
#
#
# ## Stretch Goals
# - [ ] Add your own stretch goal(s) !
# - [ ] Instead of `RidgeRegression`, try `LinearRegression`. Depending on how many features you select, your errors will probably blow up! 💥
# - [ ] Instead of `RidgeRegression`, try [`RidgeCV`](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.RidgeCV.html).
# - [ ] Learn more about feature selection:
#     - ["Permutation importance"](https://www.kaggle.com/dansbecker/permutation-importance)
#     - [scikit-learn's User Guide for Feature Selection](https://scikit-learn.org/stable/modules/feature_selection.html)
#     - [mlxtend](http://rasbt.github.io/mlxtend/) library
#     - scikit-learn-contrib libraries: [boruta_py](https://github.com/scikit-learn-contrib/boruta_py) & [stability-selection](https://github.com/scikit-learn-contrib/stability-selection)
#     - [_Feature Engineering and Selection_](http://www.feat.engineering/) by Kuhn & Johnson.
# - [ ] Try [statsmodels](https://www.statsmodels.org/stable/index.html) if you’re interested in more inferential statistical approach to linear regression and feature selection, looking at p values and 95% confidence intervals for the coefficients.
# - [ ] Read [_An Introduction to Statistical Learning_](http://faculty.marshall.usc.edu/gareth-james/ISL/ISLR%20Seventh%20Printing.pdf), Chapters 1-3, for more math & theory, but in an accessible, readable way.
# - [ ] Try [scikit-learn pipelines](https://scikit-learn.org/stable/modules/compose.html).

# + colab={} colab_type="code" id="o9eSnDYhUGD7"
import os, sys
in_colab = 'google.colab' in sys.modules

# If you're in Colab...
if in_colab:
    # Pull files from Github repo
    os.chdir('/content')
    # !git init .
    # !git remote add origin https://github.com/LambdaSchool/DS-Unit-2-Regression-Classification.git
    # !git pull origin master
    
    # Install required python packages
    # !pip install -r requirements.txt
    
    # Change into directory for module
    os.chdir('module3')

# + colab={} colab_type="code" id="ipBYS77PUwNR"
# Ignore this Numpy warning when using Plotly Express:
# FutureWarning: Method .ptp is deprecated and will be removed in a future version. Use numpy.ptp instead.
import warnings
warnings.filterwarnings(action='ignore', category=FutureWarning, module='numpy')

# + colab={} colab_type="code" id="QJBD4ruICm1m"
import pandas as pd

# Read New York City property sales data
df = pd.read_csv('../data/condos/NYC_Citywide_Rolling_Calendar_Sales.csv')

# Change column names: replace spaces with underscores
df.columns = [col.replace(' ', '_') for col in df]

# SALE_PRICE was read as strings.
# Remove symbols, convert to integer
df['SALE_PRICE'] = (
    df['SALE_PRICE']
    .str.replace('$','')
    .str.replace('-','')
    .str.replace(',','')
    .astype(int)
)

# + colab={} colab_type="code" id="xwovGo-PBzkJ"
# BOROUGH is a numeric column, but arguably should be a categorical feature,
# so convert it from a number to a string
df['BOROUGH'] = df['BOROUGH'].astype(str)

# + colab={"base_uri": "https://localhost:8080/", "height": 188} colab_type="code" id="hfpAt0hEBzkP" outputId="61520084-a9eb-44dd-8ce7-1c880e9214bf"
# Reduce cardinality for NEIGHBORHOOD feature

# Get a list of the top 10 neighborhoods
top10 = df['NEIGHBORHOOD'].value_counts()[:10].index

# At locations where the neighborhood is NOT in the top 10, 
# replace the neighborhood with 'OTHER'
df.loc[~df['NEIGHBORHOOD'].isin(top10), 'NEIGHBORHOOD'] = 'OTHER'

df.head(1)

# + colab={} colab_type="code" id="RRuH4-y2tv5K"


# + [markdown] colab_type="text" id="_YX7C6DEECy7"
# ####Changing to Pandas Date Time

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" id="lRnKrHzKBzkV" outputId="20d113d7-5822-4136-f8ee-739f59a14cd2"
df.head()
df['SALE_DATE'].dtype
df["SALE_DATE"] = pd.to_datetime(df['SALE_DATE'])
df['SALE_DATE'].dtypes

# + colab={} colab_type="code" id="zidOld8pJ2Wu"
#Changing Land Square Feet to a numeric

# print(df['LAND_SQUARE_FEET'].dtypes)
# df['LAND_SQUARE_FEET'] = df['LAND_SQUARE_FEET'].dropna()
# print(df['LAND_SQUARE_FEET'].value_counts().sum())
# df['LAND_SQUARE_FEET'] = (
#     df['LAND_SQUARE_FEET']
#     .str.replace(',' , '')
#     # .str.replace('########', 0)
#     # .astype(float)
# )
# print(df['LAND_SQUARE_FEET'].dtypes)
# df["LAND_SQUARE_FEET"].describe(exclude='number')

# + [markdown] colab_type="text" id="_0iGQPE7GH5g"
#

# + colab={"base_uri": "https://localhost:8080/", "height": 86} colab_type="code" id="b0kBzy4XELGk" outputId="1d159298-2ea8-44f3-a719-7fef8c2d09a2"
#Train set Jan-Mar
cutoff = pd.to_datetime('2019-04-01')
train = df[df['SALE_DATE'] < cutoff]
train['SALE_DATE'].dt.month.value_counts()

# + colab={"base_uri": "https://localhost:8080/", "height": 52} colab_type="code" id="1oBh7ea5GJGJ" outputId="d58f7c70-6705-4c1b-afb9-d7d9576da4b9"
#Test set April 2019
test = df[df['SALE_DATE'].dt.month == 4]
test['SALE_DATE'].dt.month.value_counts()

# + colab={} colab_type="code" id="0T2m60WdTgxM"


# + [markdown] colab_type="text" id="nhfuoM3tISEG"
# ####One Hot Encoding

# + colab={"base_uri": "https://localhost:8080/", "height": 434} colab_type="code" id="RJBLA0nBJZfb" outputId="9b918ce7-8414-4576-a86f-e9a2895f74c5"
print('Train Shape', train.shape)
print('Test Shape', test.shape)
train.isnull().sum()

# + colab={} colab_type="code" id="FNvP6sRJVNjj"
test = test.drop(labels=['EASE-MENT', 'APARTMENT_NUMBER', 'LAND_SQUARE_FEET'], axis=1)

# + colab={} colab_type="code" id="gK2u3SvrWnBh"
train = train.drop(labels=['EASE-MENT', 'APARTMENT_NUMBER', 'LAND_SQUARE_FEET'], axis=1)

# + colab={"base_uri": "https://localhost:8080/", "height": 677} colab_type="code" id="FmzJAsv7WrtV" outputId="77bad621-c963-459f-f9b3-18e2f98ab906"
#No Nulls
train = train.dropna()
test = test.dropna()
print(train.isnull().sum())
print(test.isnull().sum())

# + colab={"base_uri": "https://localhost:8080/", "height": 348} colab_type="code" id="KTCHlgMpndrr" outputId="de5efe32-bbc5-4f72-e1e5-9f423a90c912"
test = test.loc[test['BUILDING_CLASS_CATEGORY']=='01 ONE FAMILY DWELLINGS',:]
train = train.loc[train['BUILDING_CLASS_CATEGORY']=='01 ONE FAMILY DWELLINGS',:]
train = train[(train['SALE_PRICE'] >= 100000) & (train['SALE_PRICE'] <= 2000000)]
test = test[(test['SALE_PRICE'] >= 100000) & (test['SALE_PRICE'] <= 2000000)]
test.head()

# + colab={"base_uri": "https://localhost:8080/", "height": 300} colab_type="code" id="ATsdxTdRHo2X" outputId="0201a48d-221c-4903-da3a-07ecc410503e"
train.select_dtypes(exclude='number').describe().T.sort_values(by='unique')

# + colab={} colab_type="code" id="q1mBfrogOfSG"
target = 'SALE_PRICE'
high_cardinality = ['ADDRESS', 'SALE_DATE']
features = train.columns.drop([target] + high_cardinality)

X_train = train[features]
y_train = train[target]
X_test = test[features]
y_test = test[target]

# + colab={"base_uri": "https://localhost:8080/", "height": 226} colab_type="code" id="0psPHkwXPfKd" outputId="c617c64b-84a3-4d88-c744-1e240f1437f6"
X_train.head()

# + colab={} colab_type="code" id="SmXGG-1xP2kK"
import category_encoders as ce
encoder = ce.OneHotEncoder(use_cat_names=True)
X_train = encoder.fit_transform(X_train)
X_test = encoder.transform(X_test)

# + colab={"base_uri": "https://localhost:8080/", "height": 135} colab_type="code" id="5nHcNfK3Qbj2" outputId="7250bbb6-fc19-4928-838d-d642eb59c1a3"
print(X_train.shape)
X_train.head(1)

# + colab={} colab_type="code" id="xjro_E3ER59d"
X_train.isnull().sum()

# + [markdown] colab_type="text" id="yKt1N-piQuxn"
# ###Feature Selction with Select K Best

# + colab={"base_uri": "https://localhost:8080/", "height": 228} colab_type="code" id="4SddpdGNQqc9" outputId="75a539b2-fe05-47fe-c480-f2323e869334"
from sklearn.feature_selection import f_regression, SelectKBest

selector = SelectKBest(score_func=f_regression, k=15)

X_train_selected = selector.fit_transform(X_train, y_train)
X_test_selected = selector.transform(X_test)
X_train_selected.shape, X_test_selected.shape


# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" id="g879-qAkhAOk" outputId="cb30bc13-1a96-4e44-80ae-c36bb4a8bd2b"
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_absolute_error

model = LinearRegression()
model.fit(X_train_selected, y_train)
y_pred = model.predict(X_test_selected)

mae = mean_absolute_error(y_test, y_pred)

mae

# + colab={} colab_type="code" id="EMWD01jdjbBi"
for k in range(1, len(X_train.columns)+1):
    
    print(f'{k} features')
    
    selector = SelectKBest(score_func=f_regression, k=k)
    X_train_selected = selector.fit_transform(X_train, y_train)
    X_test_selected = selector.transform(X_test)
    
    model = LinearRegression()
    model.fit(X_train_selected, y_train)
    y_pred = model.predict(X_test_selected)
    
    mae = mean_absolute_error(y_test, y_pred)
    print(f'Test MAE: ${mae:,.0f} \n')

# + [markdown] colab_type="text" id="trxrUyOTah3I"
# ###Ridge Regression

# + colab={} colab_type="code" id="_P-xYGfovTyr"
test = test.drop(['SALE_PRICE'], axis=1)


# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" id="AMDzvWfaahjS" outputId="7553e945-81c0-41c7-b7c7-5c567379b862"
from sklearn.linear_model import Ridge

ridge_reg = Ridge(alpha=1).fit(X_train,y_train)
mean_absolute_error(y_test, ridge_reg.predict(X_test))

# + colab={} colab_type="code" id="v2rwjiVQq3v4"


StarcoderdataJupyterScriptsDedupFiltered