# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# + pycharm={"name": "#%%\n"}
import os
import tarfile
import urllib
# https://github.com/ageron/handson-ml2
import matplotlib.pyplot as plt

# + pycharm={"name": "#%%\n"}
# DOWNLOAD_ROOT = "https://github.com/ageron/handson-ml2"
HOUSING_PATH = os.path.join("datasets","housing")
HOUSING_URL =  "C:/Users/Donald/Desktop/handson-ml2/datasets/housing/housing.tgz"


# + pycharm={"name": "#%%\n"}
def fetch_housing_data(housing_url=HOUSING_URL,housing_path=HOUSING_PATH):
    os.makedirs(housing_path,exist_ok=True)
    tgz_path = os.path.join(housing_path,"housing.tgz")
    urllib.request.urlretrieve(housing_url,housing_path)
    housing_tgz = tarfile.open(tgz_path)
    housing_tgz.extractall(path=housing_path)
    housing_tgz.close()


# + pycharm={"name": "#%%\n"}
import pandas as pd

def load_housing_data(housing_path=HOUSING_PATH):
    csv_path = os.path.join(housing_path,"housing.csv")
    return pd.read_csv("./housing.csv")


# + pycharm={"name": "#%%\n"}
housing = load_housing_data()
housing.head()

# + pycharm={"name": "#%%\n"}
housing.info()

# + pycharm={"name": "#%%\n"}
housing.describe()

# + pycharm={"name": "#%%\n"}
housing.hist(bins=50,figsize=(20,15))
plt.show()

# + pycharm={"name": "#%%\n"}
housing.median()

# + pycharm={"name": "#%%\n"}
housing.plot()

# + pycharm={"name": "#%%\n"}
#now the technical parts come in, to plot the instances into test and training sets
import numpy as np

def split_test_and_train(data,test_ratio):
    shuffled_indices = np.random.permutation(len(data))
    test_set_size = int(len(data) * test_ratio)
    test_indices = shuffled_indices[:test_set_size]
    train_indices = shuffled_indices[test_set_size:]
    return data.iloc[train_indices],data.iloc[test_indices]


# + pycharm={"name": "#%%\n"}
a = split_test_and_train(housing,test_ratio=0.2)
print(a)

# + pycharm={"name": "#%%\n"}
train_set,test_set = split_test_and_train(housing,0.2)
len(train_set)

# + pycharm={"name": "#%%\n"}
len(test_set)

# + pycharm={"name": "#%%\n"}
from zlib import crc32

def test_check_set(identifier, test_ratio):
    return crc32(np.int64(identifier)) & 0xfffffffff < test_ratio * 2*32


# + pycharm={"name": "#%%\n"}
def split_test_train_by_id(data,id_column,test_ratio):
    ids = data[id_column]
    in_test_set = ids.apply(lambda id_ : test_check_set(id_,test_ratio))
    return data.loc[~in_test_set],data.loc[in_test_set]


# -

#

# + pycharm={"name": "#%%\n"}
housing_with_id = housing.reset_index()
train_set,test_set = split_test_train_by_id(housing_with_id,"index",0.2)

# + pycharm={"name": "#%%\n"}
housing_with_id["id"] = housing["longitude"] * 1000 + housing["latitude"]
train_set,test_set = split_test_train_by_id(housing_with_id,"id",0.2)

# + pycharm={"name": "#%%\n"}
from sklearn.model_selection import train_test_split

train_set,test_set = train_test_split(housing,test_size=0.2,random_state=42)

# + pycharm={"name": "#%%\n"}
housing["income_category"] = pd.cut(housing["median_income"],bins=[0.,1.5,3.0,4.5,6.0,np.inf],labels=[1,2,3,4,5])
housing["income_category"].hist()

# + pycharm={"name": "#%%\n"}
from sklearn.model_selection import StratifiedShuffleSplit

split = StratifiedShuffleSplit(n_splits=1,test_size=0.2,random_state=42)
for train_index, test_index  in split.split(housing,housing["income_category"]):
    strat_train_set = housing.loc[train_index]
    strat_test_set = housing.loc[test_index]


# + pycharm={"name": "#%%\n"}
strat_test_set["income_category"].value_counts() / len(strat_test_set)

# + pycharm={"name": "#%%\n"}
for set_ in (strat_train_set,strat_test_set):
    set_.drop("income_category", axis=1, inplace=True)

# + pycharm={"name": "#%%\n"}
housingCopy = strat_train_set.copy()
housingCopy.plot(kind="scatter",x="longitude",y="latitude")

# + pycharm={"name": "#%%\n"}
housingCopy.plot(kind="scatter",x="longitude",y="latitude",alpha=0.1)

# + pycharm={"name": "#%%\n"}
housingCopy.plot()

# + pycharm={"name": "#%%\n"}
housing.plot(kind = "scatter",x = "longitude",y = "latitude", alpha = 0.4, s=housing["population"]/100)

# + pycharm={"name": "#%%\n"}


# + pycharm={"name": "#%%\n"}


# + pycharm={"name": "#%%\n"}
corr_matrix = housing.corr()
corr_matrix["median_house_value"].sort_values(ascending=False)
# -

from pandas.plotting import scatter_matrix

attributes = ["median_house_value","median_income","housing_median_age","total_rooms"]
scatter_matrix(housing[attributes], figsize=(12,8))

housing.plot(kind="scatter",x="median_income",y="median_house_value",alpha=0.1)

#  ## Experimenting With Attribute Combinations:
#      Before preparing the machine learning algorithm you must try the ATTRIBUTES COMBINATIONS For example total numbers of rooms is not really useful if you don't know how many households are there.

# +
#creating new attributes

housing["rooms_per_household"]=housing["total_rooms"]/housing["households"]
housing["bedrooms_per_rooms"]=housing["total_bedrooms"]/housing["total_rooms"]
housing["population_per_household"]=housing["population"]/housing["households"]

corr_matrix = housing.corr()
corr_matrix["median_house_value"].sort_values(ascending=False)
# -

# # Prepare the data for ML algo
# ### writing the functions is ought to be more efficient.

housing = strat_train_set.drop("median_house_value", axis = 1)
housing_labels = strat_train_set["median_house_value"].copy() 

# # Data Cleaning
#     Gonna use DataFrame's 
#         dropna(),drop(),and fillna()

median = housing["total_bedrooms"].median()
housing["total_bedrooms"].fillna(median, inplace=True)

# # SimpleImputer
#     sklearn's fuction which takes care of missing values
#     

# +
from sklearn.impute import SimpleImputer

imputer = SimpleImputer(strategy = "median")
housing_num = housing.drop("ocean_proximity",axis=1)
imputer.fit(housing_num)
# -

imputer.statistics_

housing_num.median().values

x = imputer.transform(housing_num)

housing_tr = pd.DataFrame(x,columns=housing_num.columns,index=housing_num.index)

# # Scikit-learn's Design is crazy
#     estimators, transformers, Predicors, inspection
# ## Handling text and categorical Attributes
#     so far we have dealt with numerical attributes, but now let's look at text attributes

housing_category = housing[["ocean_proximity"]]
housing_category.head(10)

from sklearn.preprocessing import OrdinalEncoder
ordinal_encoder=OrdinalEncoder()
housing_cat_encoded = ordinal_encoder.fit_transform(housing_category)
housing_cat_encoded[:10]

ordinal_encoder.categories_

# +
from sklearn.preprocessing import OneHotEncoder 
cat_encoder=OneHotEncoder()

housing_cat_1hot=cat_encoder.fit_transform(housing_category)
housing_cat_1hot
# -

housing_cat_1hot.toarray()

cat_encoder.categories_

# # Custom Transformers
#     

from sklearn.base import BaseEstimator, TransformerMixin
rooms_ix, bedrooms_ix, population_ix, households_ix = 3,4,5,6


class CombinedAttributesAdder(BaseEstimator, TransformerMixin):
    def __init__(self,add_bedrooms_per_room = True):
        self.add_bedrooms_per_room = add_bedrooms_per_room
    def fit(self,X,y=None):
        return self
    def transform(self,X,y=None):
        rooms_per_household = X[:, rooms_ix] / X[:, households_ix]
        population_per_household = X[:, population_ix] / X[:, households_ix]
        if self.add_bedrooms_per_room:
            bedrooms_per_room = X[:, bedrooms_ix] / X[:, rooms_ix]
            return np.c_[X,rooms_per_household,population_per_household,bedrooms_per_room]
        else:
            return np.c_[X, rooms_per_household, population_per_household]
attr_adder = CombinedAttributesAdder(add_bedrooms_per_room=False)
housing_extra_attribs = attr_adder.transform(housing.values)

# # Feature Scaling
# ## Transformation Pipelines
#

# +
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler

num_pipeline = Pipeline([
    ('imputer', SimpleImputer(strategy="median")),
    ('attribs_adder',CombinedAttributesAdder()),
    ('std_scaler', StandardScaler()),
])

housing_num_tr = num_pipeline.fit_transform(housing_num)

# +
from sklearn.compose import ColumnTransformer

num_attribs = list(housing_num)
cat_attribs = ["ocean_proximity"]

full_pipeline = ColumnTransformer([
    ("num",num_pipeline,num_attribs),
    ("cat",OneHotEncoder(),cat_attribs),
])

housing_prepared = full_pipeline.fit_transform(housing)
# -

# # Select and Train a Model
# ## Training and Evaluating on the Training_Set

from sklearn.linear_model import LinearRegression
lin_reg = LinearRegression()
lin_reg.fit(housing_prepared, housing_labels)


some_data = housing.iloc[:5]
some_labels = housing_labels.iloc[:5]
some_data_prepared = full_pipeline.transform(some_data)

print("Predictions:", lin_reg.predict(some_data_prepared))

print("Labels:", list(some_labels))

# +
from sklearn.metrics import mean_squared_error

housing_predictions = lin_reg.predict(housing_prepared)
lin_mse = mean_squared_error(housing_labels, housing_predictions)
lin_rmse = np.sqrt(lin_mse)
lin_rmse
# -

# # Ways to fix underfitting :
#     1>reduce the constraints
#     2>Use more powerfull model to feed the Training algo.
#     
# ## using decision tree

# +
from sklearn.tree import DecisionTreeRegressor

tree_reg = DecisionTreeRegressor()
tree_reg.fit(housing_prepared, housing_labels)
# -

housing_predictions = tree_reg.predict(housing_prepared)
tree_mse = mean_squared_error(housing_labels, housing_predictions)
tree_rmse = np.sqrt(tree_mse)
tree_rmse


# # better evaluation via Cross-Validation
#     using train_test_split() function to split the training set into smaller training and validation set
#     
# ## Alternative :
#     We ca  use scikit-learn's **K-fold** cross validation feature which divides into 10 distinct sub-sets

# +
from sklearn.model_selection import cross_val_score
scores = cross_val_score(tree_reg, housing_prepared, housing_labels, scoring = 'neg_mean_squared_error',cv = 10)

tree_rmse_scores = np.sqrt(-scores)


# -

def display_scores(scores):
    print("scores",scores)
    print("mean",scores.mean())
    print("standard deviation",scores.std())


display_scores(tree_rmse_scores)

# # We will now try The RandomForest regressor

# +
from sklearn.ensemble import RandomForestRegressor

forest_reg = RandomForestRegressor()
forest_reg.fit(housing_prepared, housing_labels)
# -

housing_predictions = forest_reg.predict(housing_prepared)
forest_mse =mean_squared_error(housing_labels, housing_predictions)
forest_rmse =np.sqrt(forest_mse)
forest_rmse

forest_scores =cross_val_score(lin_reg,housing_prepared, housing_labels, scoring = "neg_mean_squared_error",cv=10)
forest_rmse_scores = np.sqrt(-forest_scores)
display_scores(forest_rmse_scores)

# # Time to Fine_Tune my Model

# ## fine-tuning can be done in two ways :
#     grid_search / randomized_search
#     for small hyperparameter space / for large hyperparameter spacefor 

# +
## grid search first
### Here bootstrap is set to False as now it will consider combination's sum instead .
#### like 12+6 = 18
from sklearn.model_selection import GridSearchCV

param_grid = [
    {"n_estimators":[3,30,300],"max_features":[2,4,6,10]},
    {'bootstrap':[False],'n_estimators':[3,20],'max_features':[2,3,6]},
]

forest_reg = RandomForestRegressor()

grid_search = GridSearchCV(forest_reg,param_grid,cv=5,
                      scoring='neg_mean_squared_error',
                      return_train_score = True)
grid_search.fit(housing_prepared, housing_labels)
# -

grid_search.best_params_

grid_search.best_estimator_

cvres = grid_search.cv_results_
for mean_score, params in zip(cvres["mean_test_score"],cvres["params"]):
    print(np.sqrt(-mean_score),params)



StarcoderdataJupyterScriptsDedupFiltered