<jupyter_start><jupyter_text><jupyter_code>from google.colab import drive
drive.mount('/content/drive')
# Ignore the warnings
import warnings
warnings.filterwarnings('ignore')

# System related and data input controls
import os

# Data manipulation, visualization and useful functions
import pandas as pd
import numpy as np
from itertools import product # iterative combinations
from tqdm import tqdm
import matplotlib.pyplot as plt
import seaborn as sns

# Modeling algorithms
# General(Statistics/Econometrics)
from sklearn import preprocessing
import statsmodels.api as sm
import statsmodels.tsa.api as smt
import statsmodels.formula.api as smf
from statsmodels.stats.outliers_influence import variance_inflation_factor
from scipy import stats

# Regression
from sklearn.linear_model import LinearRegression, Ridge, Lasso, ElasticNet
from sklearn.kernel_ridge import KernelRidge
from sklearn.neighbors import KNeighborsRegressor
from sklearn.svm import SVR
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor, BaggingRegressor, GradientBoostingRegressor, AdaBoostRegressor
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor

# Classification
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import LinearSVC, SVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier

# Model selection
from sklearn.model_selection import train_test_split,cross_validate
from sklearn.model_selection import KFold
from sklearn.model_selection import GridSearchCV

# Evaluation metrics
# for regression
from sklearn.metrics import mean_squared_log_error, mean_squared_error,  r2_score, mean_absolute_error
# for classification
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score

import pandas as pd
from keras.models import Sequential, Model, load_model
from keras.layers import Input, Dense, Activation, Flatten, Dropout
from keras.layers import SimpleRNN, LSTM, GRU
item_categories = pd.read_csv('/content/drive/MyDrive/data/item_categories.csv')
items = pd.read_csv('/content/drive/MyDrive/data/items.csv')
#sales_train = pd.read_csv('/content/drive/MyDrive/data/sales_train.csv') #학습데이터
sample_submission = pd.read_csv('/content/drive/MyDrive/data/sample_submission.csv')
shops = pd.read_csv('/content/drive/MyDrive/data/shops.csv')
test = pd.read_csv('/content/drive/MyDrive/data/test.csv')#테스트 데이터

raw_data = pd.read_csv('/content/drive/MyDrive/data/sales_train.csv' ) #학습데이터

raw_data
df = raw_data.copy()
#1.train에 중복값 제거 (train의 모든 컬럼값이 동일한 value를 삭제)
# drop_duplicates
subset= ['date','date_block_num','shop_id','item_id','item_cnt_day']
print(df.duplicated(subset=subset).value_counts())
df.drop_duplicates(subset=subset, inplace=True)
# 2935849 - 2935825 = 24개의 중복데이터 삭제
#2.test에 있는 세일즈수만 예측하면되기때문에 train에서 test에 없는 상품 제거
#test_shops = test.shop_id.unique()
#test_items = test.item_id.unique()
#df = df[df.shop_id.isin(test_shops)]
#df = df[df.item_id.isin(test_items)]
#df.shape
# 기존2935825에서1224429으로 절반 이상 데이터 드랍
df2 = df
df2 = df.rename_axis('ID2').reset_index()
df2
df2= df2[(df2['item_cnt_day'] < df2['item_cnt_day'].quantile(0.95))]
df2['date'] = pd.to_datetime(df2['date'])
df2
df2 = df2.drop('ID2',axis=1)
df2['date'] = df2['date'].sort_values()
df2 = df2.reset_index()
df2 = df2.drop('index', axis=1)
df2 = df2.rename_axis('ID').reset_index()
df2
df2.hist()
df2['item_cnt_day'].hist()
df2['item_cnt_day'].quantile(0.95)
df2.head()
df2.info()
df2['item_cnt_day'].hist()
df2
df2 = df2.drop(['date_block_num'],axis=1)
df2 = df2.drop(['item_price'],axis=1)
df2
df3 = df2.set_index('date')
df3.head()
df4 = df3.groupby(by=['shop_id', 'item_id'])['item_cnt_day'].resample('M').sum()
df5 = pd.DataFrame(df4)
df5 = df5.reset_index()
df5 = df5.drop('date',axis=1)
df5 = df5.rename_axis('ID').reset_index()
df5
items.head()
df6 = pd.merge(df5, items, how='left', left_on='item_id',right_on='item_id')
df6 = df6.drop('item_name',axis=1)
y = df6.loc[:,'item_cnt_day']
X = df6.drop(['item_cnt_day'], axis=1)
X
#df_test_orig = df3.loc[:, ['shop_id', 'item_id', 'item_cnt_day']]
y
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=1)

# 학습과 테스트 데이터 분리
#split = "2015-01-01"
#df_train = df3[:split]
#df_test = df3[split:]

#df_train_y = df_train.loc[:,'item_cnt_day']
#df_train_x = df_train.drop('item_cnt_day', axis=1)
#df_test_y = df_test.loc[:,'item_cnt_day']
#df_test_x = df_test.drop('item_cnt_day', axis=1)
# 나중에 예측값과 비교하기 위해 test 데이터 복제본 저장
#df_test_orig = df_test.loc[:, ['shop_id', 'item_id', 'item_cnt_day']]
XG_model_month = XGBRegressor()
XG_model_month.fit(X_train, y_train,
                   eval_set=[(X_test, y_test)],
                   early_stopping_rounds=10,verbose=False)

from sklearn.metrics import mean_absolute_error, mean_squared_error
from math import sqrt
pred = XG_model_month.predict(X_test)
print(mean_absolute_error(y_test, pred))
print(sqrt(mean_squared_error(y_test, pred)))
# 주요하게 적용하는 변수를 판단
from xgboost import plot_importance

import matplotlib.pyplot as plt

%matplotlib inline

plot_importance(XG_model_month, height=0.9)
#xgboost = XG_model_month.predict(df3)

# 실제 값과 예측 값을 하나의 DataFrame으로 만들고 visualize
#result=pd.concat([df_test_orig[(df_test_orig.store==1)&(df_test_orig.item==2)].reset_index(), pd.DataFrame(xgboost, columns=['xgboost'])], axis=1, ignore_index=False)
#result = result.set_index('date')
#result = result.loc[:, ['item_cnt_day', 'xgboost']]
#result.plot()
test2=test.copy()
items.head()
test2 = pd.merge(test2,items, how='left', left_on='item_id',right_on='item_id')
test2=test2.drop('item_name', axis=1)
test2.info()
test2.astype(float)
pred = XG_model_month.predict(test2)
submission = pd.DataFrame(pred)
submission_copy = submission.rename_axis('ID').reset_index()

submission_copy
submission_copy.columns = ['ID', 'item_cnt_month']
submission_copy
submission_copy.mean()

#submission_copy['item_cnt_month'] = submission_copy['item_cnt_month'].astype(float)
submission_copy.to_csv('XG_submission_final.csv',index=False)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup