<jupyter_start><jupyter_text>Load and Merge Data<jupyter_code>onedrive_path = "C:/Users/cfowle/The Estée Lauder Companies Inc/TeamAnis - General/"
reviews = pd.read_csv(onedrive_path + "Data/Ratings and Reviews/reviews_demand_subcat.csv")
cc      = pd.read_csv(onedrive_path + "Data/Consumer Care/cc_product_month.csv")
products = pd.read_csv(onedrive_path + "Data/Product/product_codes.csv")
print(len(reviews))
reviews = reviews.drop("item_description", axis = 1).drop_duplicates()
print(len(reviews))
cc = cc.rename(columns = {"Date Month": "date", "P4": "itemid_4", "Brand Clean": "elc_brand"})
products = products.rename(columns = {"P4": "itemid_4", "brand": "elc_brand", "SubCategory":"sub_category"})
products = products[["elc_brand", "itemid_4", 'Major Category', 'Application', 'Category',
       'sub_category']].drop_duplicates(["elc_brand", "itemid_4"])
cc_product = cc.merge(products, how = "left", on = ["itemid_4", "elc_brand"])
cc_reviews_product = reviews.merge(cc_product, how = "left").fillna(0)
cc_reviews_product = cc_reviews_product.groupby(["elc_brand", "date", "sub_category"]).sum().reset_index()<jupyter_output><empty_output><jupyter_text>Feature Engineering Target Encoding<jupyter_code>subcat_map = cc_reviews_product.groupby("sub_category")["demand"].mean().reset_index()
brand_map  = cc_reviews_product.groupby("elc_brand")["demand"].mean().reset_index()

cc_reviews_product["month"] = [x[-2:] for x in cc_reviews_product["date"].values]
cc_reviews_product["year"]  = [x[:4] for x in cc_reviews_product["date"].values]

month_map = cc_reviews_product.groupby("month")["demand"].mean().reset_index()
year_map  = cc_reviews_product.groupby("year")["demand"].mean().reset_index()
subcat_map = subcat_map.rename(columns = {"demand":"subcat_avg_demand"})
month_map  = month_map.rename(columns = {"demand":"month_avg_demand"})
year_map   = year_map.rename(columns = {"demand":"year_avg_demand"})
brand_map  = brand_map.rename(columns = {"demand":"brand_avg_demand"})
cc_reviews_product = cc_reviews_product.merge(subcat_map).merge(month_map).merge(year_map).merge(brand_map)
cc_reviews_product = cc_reviews_product.sort_values("date")<jupyter_output><empty_output><jupyter_text>Create dataset without customer data<jupyter_code>non_customer_data = cc_reviews_product[["date", "demand_F1", "demand", "demand_P2", "demand_P1",
                                   "subcat_avg_demand", "month_avg_demand", "year_avg_demand", "brand_avg_demand",
                                   "month", "year"]]<jupyter_output><empty_output><jupyter_text>Create Dataset with only Relative Measures<jupyter_code>relative_data = cp.deepcopy(non_customer_data)
subcat = cc_reviews_product ##just here so I can copy some old code easily
relative_data["percent_1"] = subcat["rating_1"]/subcat["nb_reviews"]
relative_data["percent_2"] = subcat["rating_2"]/subcat["nb_reviews"]
relative_data["percent_3"] = subcat["rating_3"]/subcat["nb_reviews"]
relative_data["percent_4"] = subcat["rating_4"]/subcat["nb_reviews"]
relative_data["percent_5"] = subcat["rating_5"]/subcat["nb_reviews"]
relative_data["percent_negative"] = subcat['sentiment_negative']/subcat["nb_reviews"]
relative_data["percent_neutral"] = subcat['sentiment_neutral']/subcat["nb_reviews"]
relative_data["percent_positive"] = subcat['sentiment_positive']/subcat["nb_reviews"]
relative_data = relative_data.merge(subcat_map).merge(month_map).merge(year_map).merge(brand_map)
relative_data = relative_data.sort_values("date").reset_index(drop=True)
relative_data = relative_data.fillna(0)
full_relative_data = cp.deepcopy(relative_data)
subcat["cc_tot"] = subcat['Complaints'] + subcat['Compliments'] + subcat['Questions'] + subcat['Suggestions']
cc_counts = cc.drop(['Product Code', 'itemid_4', 'elc_brand', "date"], axis = 1).columns
for col in cc_counts:
    full_relative_data["percent_" + col] = subcat[col]/subcat["cc_tot"]
full_relative_data = full_relative_data.fillna(0)
full_relative_data<jupyter_output><empty_output><jupyter_text>Build Model with Customer Data<jupyter_code>X = relative_data.drop(["demand_F1", "date", "sub_category", "elc_brand"], axis = 1)
y = relative_data[["demand_F1"]]
X.columns
X = StandardScaler().fit_transform(X)
pca = PCA().fit(X)
plt.plot(np.cumsum(pca.explained_variance_ratio_))
plt.xlabel('number of components')
plt.ylabel('cumulative explained variance')
pca = PCA(n_components=10)
principalComponents = pca.fit_transform(X)
principalDf = pd.DataFrame(data = principalComponents)
#principalDf.to_csv(onedrive_path + "Output/2020_06_22_to_rama/pca.csv")
X_train, X_test, y_train, y_test = train_test_split(principalDf, y, shuffle = False)
tree = DecisionTreeRegressor()
tree.fit(X_train, y_train)
tree_param = {'max_depth': [2, 3, 5, 7],
             'min_samples_split':  [2, 4, 18, 16, 32, 64],
             'min_samples_leaf': [2, 4, 18, 16, 32, 64]}
clf = GridSearchCV(tree, tree_param, cv=5)
clf.fit(X_train, y_train)
clf.best_score_
pred = clf.best_estimator_.predict(X_test)
metrics.r2_score(pred, y_test)<jupyter_output><empty_output><jupyter_text>What if we remove Demand Lags<jupyter_code>no_lags = relative_data[['subcat_avg_demand',
       'month_avg_demand', 'year_avg_demand', 'brand_avg_demand', 'month',
       'year', 'percent_1', 'percent_2', 'percent_3', 'percent_4', 'percent_5',
       'percent_negative', 'percent_neutral', 'percent_positive']]
X = StandardScaler().fit_transform(no_lags)
pca = PCA().fit(X)
plt.plot(np.cumsum(pca.explained_variance_ratio_))
plt.xlabel('number of components')
plt.ylabel('cumulative explained variance')
pca = PCA(n_components=10)
principalComponents = pca.fit_transform(X)
principalDf = pd.DataFrame(data = principalComponents)
X_train, X_test, y_train, y_test = train_test_split(principalDf, y, shuffle = False)
tree = DecisionTreeRegressor()
tree.fit(X_train, y_train)

tree_param = {'max_depth': [2, 3, 5, 7],
             'min_samples_split':  [2, 4, 18, 16, 32, 64],
             'min_samples_leaf': [2, 4, 18, 16, 32, 64]}
clf = GridSearchCV(tree, tree_param, cv=5)
clf.fit(X_train, y_train)
clf.best_score_
pred = clf.best_estimator_.predict(X_test)
metrics.r2_score(pred, y_test)<jupyter_output><empty_output><jupyter_text>What if I didn't do PCA?<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(no_lags, y, shuffle = False)
tree = DecisionTreeRegressor()
tree.fit(X_train, y_train)

tree_param = {'max_depth': [2, 3, 4],
             'min_samples_split':  [2, 4, 18, 16, 32, 64],
             'min_samples_leaf': [2, 4, 18, 16, 32, 64]}
clf = GridSearchCV(tree, tree_param, cv=5)
clf.fit(X_train, y_train)
clf.best_score_
pred = clf.best_estimator_.predict(X_test)
metrics.r2_score(pred, y_test)
fig = plt.figure(figsize=(25,20))
_ = plot_tree(clf.best_estimator_,
            feature_names=no_lags.columns,
            filled=True)<jupyter_output><empty_output><jupyter_text>What if we removed all demand data<jupyter_code>no_demand = relative_data[['month','year', 'percent_1', 'percent_2', 'percent_3', 'percent_4', 'percent_5',
       'percent_negative', 'percent_neutral', 'percent_positive']] 
X = StandardScaler().fit_transform(no_demand)
pca = PCA().fit(X)
plt.plot(np.cumsum(pca.explained_variance_ratio_))
plt.xlabel('number of components')
plt.ylabel('cumulative explained variance')
pca = PCA(n_components=7)
principalComponents = pca.fit_transform(X)
principalDf = pd.DataFrame(data = principalComponents)
X_train, X_test, y_train, y_test = train_test_split(principalDf, y, shuffle = False)
tree = DecisionTreeRegressor()
tree.fit(X_train, y_train)

tree_param = {'max_depth': [2, 3, 5, 7],
             'min_samples_split':  [2, 4, 18, 16, 32, 64],
             'min_samples_leaf': [2, 4, 18, 16, 32, 64]}
clf = GridSearchCV(tree, tree_param, cv=5)
clf.fit(X_train, y_train)
clf.best_score_
pred = clf.best_estimator_.predict(X_test)
metrics.r2_score(pred, y_test)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup