# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + id="Z03qx2oKw0CO" colab_type="code" colab={}
# #!pip install eli5

# + id="ovmRP6uVw2X6" colab_type="code" outputId="5e56af78-bd2d-44e0-e0ff-4e9456e49b2f" executionInfo={"status": "ok", "timestamp": 1581682473380, "user_tz": -60, "elapsed": 12416, "user": {"displayName": "Mateusz \u017bukowski", "photoUrl": "", "userId": "04666613203844295870"}} colab={"base_uri": "https://localhost:8080/", "height": 168}
import pandas as pd
import numpy as np
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor

from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import cross_val_score

import eli5
from eli5.sklearn import PermutationImportance

from ast import literal_eval
from tqdm import tqdm_notebook

# + id="m6odCqaEG-iw" colab_type="code" outputId="f2027afa-a813-4947-c289-f62ca2673aaa" executionInfo={"status": "ok", "timestamp": 1581682488135, "user_tz": -60, "elapsed": 1625, "user": {"displayName": "Mateusz \u017bukowski", "photoUrl": "", "userId": "04666613203844295870"}} colab={"base_uri": "https://localhost:8080/", "height": 34}
# cd '/content/drive/My Drive/Colab Notebooks/dw_matrix'

# + id="E5gjHZvjHRxS" colab_type="code" outputId="acaf792c-bd6d-478a-d7b9-da849cdc39d9" executionInfo={"status": "ok", "timestamp": 1581682498857, "user_tz": -60, "elapsed": 7069, "user": {"displayName": "Mateusz \u017bukowski", "photoUrl": "", "userId": "04666613203844295870"}} colab={"base_uri": "https://localhost:8080/", "height": 34}
# ls data

# + id="uHC-STIfHTJh" colab_type="code" colab={}
df = pd.read_csv('data/men_shoes.csv', low_memory=False)


# + id="pQDA4tM-Hl_p" colab_type="code" colab={}
def run_model(feats, model = DecisionTreeRegressor(max_depth=5)):
  X = df[ feats ].values
  y = df['prices_amountmin'].values

  scores = cross_val_score(model, X, y, scoring='neg_mean_absolute_error')
  return np.mean(scores), np.std(scores)


# + id="tase49BLH2vj" colab_type="code" colab={}
df['brand_cat'] = df.brand.map(lambda x: str(x).lower()).factorize()[0]

# + id="jRAFdSrsICEZ" colab_type="code" outputId="81677e98-dd66-4f3d-a749-28e7e4638b5d" executionInfo={"status": "ok", "timestamp": 1581682500852, "user_tz": -60, "elapsed": 3959, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "04666613203844295870"}} colab={"base_uri": "https://localhost:8080/", "height": 34}
run_model(['brand_cat'])

# + id="uEMn6VhWIIdI" colab_type="code" outputId="7e76f6a6-3a19-4006-ba6c-59d9032d23ac" executionInfo={"status": "ok", "timestamp": 1581682503525, "user_tz": -60, "elapsed": 4533, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "04666613203844295870"}} colab={"base_uri": "https://localhost:8080/", "height": 34}
model = RandomForestRegressor(max_depth=5, n_estimators=100, random_state=0)
run_model(['brand_cat'], model)

# + id="kDL2Tve5Iuqi" colab_type="code" outputId="98d8515e-498a-435e-953d-14fb1cb87b91" executionInfo={"status": "ok", "timestamp": 1581682507411, "user_tz": -60, "elapsed": 1714, "user": {"displayName": "Mateusz \u017bukowski", "photoUrl": "", "userId": "04666613203844295870"}} colab={"base_uri": "https://localhost:8080/", "height": 173}
df.features.values


# + id="uCIk2LcOI9IR" colab_type="code" colab={}
def parse_features(x):
  output_dict = {}
  if str(x) == 'nan' : return output_dict

  features =  literal_eval(x.replace('\\"','"'))
  for item in features:
    key = item['key'].lower().strip()
    value = item['value'][0].lower().strip()

    output_dict[key] = value

  return output_dict

df['features_parsed'] = df['features'].map(parse_features)

# + id="XqI-cbrxSV3Q" colab_type="code" outputId="5fc1242d-17c6-457a-cbcd-300dcf83f31c" executionInfo={"status": "ok", "timestamp": 1581682509508, "user_tz": -60, "elapsed": 1989, "user": {"displayName": "Mateusz \u017bukowski", "photoUrl": "", "userId": "04666613203844295870"}} colab={"base_uri": "https://localhost:8080/", "height": 34}
keys = set()

# df['features_parsed'].map( lambda x: keys.update(x.keys()) )
df['features_parsed'].map( lambda x: keys.update(x.keys()) )

len(keys)


# + id="iJK5BtyBS_zu" colab_type="code" outputId="631a2b15-cbab-465b-ca0f-498fe12ae2b1" executionInfo={"status": "ok", "timestamp": 1581682513817, "user_tz": -60, "elapsed": 4659, "user": {"displayName": "Mateusz \u017bukowski", "photoUrl": "", "userId": "04666613203844295870"}} colab={"base_uri": "https://localhost:8080/", "height": 66, "referenced_widgets": ["51ee90867947452db562527d41a94004", "c20db350e88b4065a208bac78699c142", "debd164b64ed46c88e32cfb330a8795f", "7485d4e095e246dbb271f617ab63f1fa", "<KEY>", "8d671e3f28b344e1a6f18863e3587c89", "<KEY>", "<KEY>"]}
def get_name_feat(key):
  return 'feat_' + key

for key in tqdm_notebook( keys):
  df[get_name_feat(key)] = df.features_parsed.map(lambda feats: feats[key] if key in feats else np.nan)

# + id="ACjIX79ZW2cb" colab_type="code" outputId="ee88a986-e2cd-411a-a273-01e71a1dc9fa" executionInfo={"status": "ok", "timestamp": 1581682516210, "user_tz": -60, "elapsed": 764, "user": {"displayName": "Mateusz \u017bukowski", "photoUrl": "", "userId": "04666613203844295870"}} colab={"base_uri": "https://localhost:8080/", "height": 153}
df.columns

# + id="ab71HJUMaIZV" colab_type="code" colab={}
keys_stat = {}

for key in keys:
  keys_stat[key] = df[ False == df[get_name_feat(key)].isnull() ].shape[0]/df.shape[0] * 100

# + id="jregWCHaa0dT" colab_type="code" outputId="a55ef616-8483-4133-c728-2d3cfe446247" executionInfo={"status": "ok", "timestamp": 1581682522125, "user_tz": -60, "elapsed": 1951, "user": {"displayName": "Mateusz \u017bukowski", "photoUrl": "", "userId": "04666613203844295870"}} colab={"base_uri": "https://localhost:8080/", "height": 102}
{k:v for k,v in keys_stat.items() if v > 30}

# + id="ipAMxx8Oksde" colab_type="code" outputId="1a559165-093f-43a1-8735-851d85bf56be" executionInfo={"status": "ok", "timestamp": 1581682522131, "user_tz": -60, "elapsed": 1417, "user": {"displayName": "Mateusz \u017bukowski", "photoUrl": "", "userId": "04666613203844295870"}} colab={"base_uri": "https://localhost:8080/", "height": 1000}
keys_stat

# + id="Z_lIcmvNa2Pr" colab_type="code" outputId="e6ba527e-f4de-4bcd-ed3d-665963f42a8d" executionInfo={"status": "ok", "timestamp": 1581682526480, "user_tz": -60, "elapsed": 2008, "user": {"displayName": "<NAME>017bukowski", "photoUrl": "", "userId": "04666613203844295870"}} colab={"base_uri": "https://localhost:8080/", "height": 1000}
# df['feat_brand_cat'] = df.feat_brand.factorize()[0]
# df['feat_color_cat'] = df.feat_color.factorize()[0]
# df['feat_gender_cat'] = df.feat_gender.factorize()[0]
# df['feat_manufacturer part number_cat'] = df['feat_manufacturer part number'].factorize()[0]
# df['feat_material_cat'] = df.feat_material.factorize()[0]

# df['feat_sport_cat'] = df.feat_sport.factorize()[0]
# df['feat_style_cat'] = df.feat_style.factorize()[0]

for key in keys_stat:
  df[get_name_feat(key) + '_cat'] = df[get_name_feat(key)].factorize()[0]
  print(get_name_feat(key))
  # get_name_feat(key)

# + id="Jy7qEoHlcP0l" colab_type="code" colab={}
feats = ['brand_cat']

# + id="kQwFFEH0b7sG" colab_type="code" outputId="b3243d3b-ddc8-4579-e2dc-68449cc4f31d" executionInfo={"status": "ok", "timestamp": 1581682533012, "user_tz": -60, "elapsed": 3505, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "04666613203844295870"}} colab={"base_uri": "https://localhost:8080/", "height": 34}


model = RandomForestRegressor(max_depth=5, n_estimators=100)
run_model(['brand_cat'], model=model)

# + id="yWNymkyOcz8D" colab_type="code" outputId="4abbb300-3957-4ca4-d901-4c55652e18c4" executionInfo={"status": "ok", "timestamp": 1581682538755, "user_tz": -60, "elapsed": 8107, "user": {"displayName": "Mateusz \u017bukowski", "photoUrl": "", "userId": "04666613203844295870"}} colab={"base_uri": "https://localhost:8080/", "height": 34}
feats = ['brand_cat', 'feat_brand_cat', 'feat_color_cat', 'feat_gender_cat', 'feat_manufacturer part number_cat', 'feat_material_cat']
run_model(feats, model)

# + id="iYaZCchDeWWO" colab_type="code" outputId="7f0c93e9-b8b3-4e00-e968-ada27fcd2859" executionInfo={"status": "ok", "timestamp": 1581682542706, "user_tz": -60, "elapsed": 9897, "user": {"displayName": "Mateusz \u017bukowski", "photoUrl": "", "userId": "04666613203844295870"}} colab={"base_uri": "https://localhost:8080/", "height": 34}
feats = ['brand_cat', 'feat_brand_cat', 'feat_gender_cat', 'feat_material_cat', 'feat_style_cat']
run_model(feats, model)

# + id="WHAZSMyZjbZu" colab_type="code" colab={}
feats_cat = [x for x in df.columns if '_cat' in x]

# + id="RmQ1v6C7kBzE" colab_type="code" colab={}
feats_cat.remove('feat_catalog')

# + id="kRfL-Do-jlyL" colab_type="code" outputId="2bef6206-fdb3-4c8a-a69d-af5d3999f30d" executionInfo={"status": "ok", "timestamp": 1581683027631, "user_tz": -60, "elapsed": 72573, "user": {"displayName": "Mateusz \u017bukowski", "photoUrl": "", "userId": "04666613203844295870"}} colab={"base_uri": "https://localhost:8080/", "height": 34}
feats += feats_cat
feats = list(set(feats))
run_model(feats_cat, model)

# + id="gHGySEOUdGiS" colab_type="code" outputId="8ad1a2d4-abcd-480f-c358-ebfab6747b79" executionInfo={"status": "ok", "timestamp": 1581683280067, "user_tz": -60, "elapsed": 252411, "user": {"displayName": "Mateusz \u017bukowski", "photoUrl": "", "userId": "04666613203844295870"}} colab={"base_uri": "https://localhost:8080/", "height": 391}
X = df [ feats ].values
y = df['prices_amountmin'].values
m = model.fit(X, y)

perm = PermutationImportance(m, random_state=0).fit(X, y);
eli5.show_weights(perm, feature_names=feats)

StarcoderdataJupyterScriptsDedupFiltered