<jupyter_start><jupyter_text>Rossmann  Data preparation To create the feature-engineered train_clean and test_clean from the Kaggle competition data, run `rossman_data_clean.ipynb`. One important step that deals with time series is this:```pythonadd_datepart(train, "Date", drop=False)add_datepart(test, "Date", drop=False)```<jupyter_code>path = Path('../data/rossmann/')
train_df = pd.read_pickle(path/'train_clean')
train_df.head().T
n = len(train_df); n<jupyter_output><empty_output><jupyter_text>Preparing full data set<jupyter_code>train_df = pd.read_pickle(path/'train_clean')
test_df = pd.read_pickle(path/'test_clean')
len(train_df),len(test_df)
procs=[FillMissing, Categorify, Normalize]
cat_vars = ['Store', 'DayOfWeek', 'Year', 'Month', 'Day', 'StateHoliday', 'CompetitionMonthsOpen',
    'Promo2Weeks', 'StoreType', 'Assortment', 'PromoInterval', 'CompetitionOpenSinceYear', 'Promo2SinceYear',
    'State', 'Week', 'Events', 'Promo_fw', 'Promo_bw', 'StateHoliday_fw', 'StateHoliday_bw',
    'SchoolHoliday_fw', 'SchoolHoliday_bw']

cont_vars = ['CompetitionDistance', 'Max_TemperatureC', 'Mean_TemperatureC', 'Min_TemperatureC',
   'Max_Humidity', 'Mean_Humidity', 'Min_Humidity', 'Max_Wind_SpeedKm_h', 
   'Mean_Wind_SpeedKm_h', 'CloudCover', 'trend', 'trend_DE',
   'AfterStateHoliday', 'BeforeStateHoliday', 'Promo', 'SchoolHoliday']
dep_var = 'Sales'
df = train_df[cat_vars + cont_vars + [dep_var,'Date']].copy()
test_df['Date'].min(), test_df['Date'].max()
len(test_df)
cut = train_df['Date'][(train_df['Date'] == train_df['Date'][len(test_df)])].index.max()
cut
valid_idx = range(cut)
val_df = df.iloc[valid_idx]

#check validation dataframe for data not in train df
def check_val_df():
    print('checking val vs df classes')
    col_diffs={}
    for col in df:
        #new set with elements in val_df but not in df
        diffs = set(val_df[col]).difference(set(df[col]))
        print(f'col: {col}, diffs: {len(diffs)}')
        if len(diffs)>0:
            col_diffs[col]=diffs
    return col_diffs
df[dep_var].head()
def unique_deps(x:Series)->List:
    od = OrderedDict.fromkeys(x)
    res = list(OrderedDict.fromkeys(x).keys())
    res.sort()
    return res, od
classes, od =unique_deps(df[dep_var].values)
#classes
#data = TabularDataBunch.from_df(path, df=df, dep_var=dep_var, valid_idx=valid_idx, procs=procs, cat_names=cat_vars, cont_names=cont_vars, classes=classes, test_df=test_df)
#error at predict
#data = (TabularList.from_df(df, path=path, cat_names=cat_vars, cont_names=cont_vars, procs=procs, test_df=test_df) .split_by_idx(valid_idx) .label_from_df(cols=dep_var, label_cls=FloatList, log=True, classes=classes) .databunch())
#error on predict
#data = (TabularList.from_df(df, path=path, cat_names=cat_vars, cont_names=cont_vars, procs=procs, test_df=test_df)
#                   .split_by_idx(valid_idx)
#                   .label_from_df(cols=dep_var, label_cls=FloatList, log=True)
#                   .databunch())
#original method
data = (TabularList.from_df(df, path=path, cat_names=cat_vars, cont_names=cont_vars, procs=procs, test_df=test_df)
                   .split_by_idx(valid_idx)
                   .label_from_df(cols=dep_var, label_cls=FloatList, log=True)
                   .databunch())<jupyter_output>--label_from_df() cols: Sales
--label_from_df() cols: Sales<jupyter_text>Model<jupyter_code>max_log_y = np.log(np.max(train_df['Sales'])*1.2)
y_range = torch.tensor([0, max_log_y], device=defaults.device)
learn = tabular_learner(data, layers=[1000,500], ps=[0.001,0.01], emb_drop=0.04, 
                        y_range=y_range, metrics=exp_rmspe)
#learn.model
len(data.train_ds.cont_names)
learn.lr_find()
learn.recorder.plot()
#minimum at 1.0 to 3.0
learn.fit_one_cycle(3, 3e-2, wd=0.2)
learn.save('1_tabularlist')
learn.recorder.plot_losses(last=-1)
#learn.load('1');
#learn.fit_one_cycle(5, 3e-4)
#learn.fit_one_cycle(5, 3e-4)<jupyter_output><empty_output><jupyter_text>(10th place in the competition was 0.108)  Predict<jupyter_code>preds, y = learn.get_preds(DatasetType.Test)<jupyter_output><empty_output><jupyter_text>Test 2 using:data = (TabularList.from_df(df, path=path, cat_names=cat_vars, cont_names=cont_vars, procs=procs, test_df=test_df)                   .split_by_idx(valid_idx)                   .label_from_df(cols=dep_var, label_cls=FloatList, log=True, classes=classes)                   .databunch())/mnt/963GB/Data/Python/Courses/fastai/fastai/fastai/basic_train.py in get_preds(model=TabularModel(  (embeds): ModuleList(    (0): E...in_features=500, out_features=1, bias=True)  )), dl=None, pbar=None, cb_handler=CallbackHandler(callbacks=[], metrics=[], beta=0.98), activ=, loss_func=None, n_batch=None)     36     "Tuple of predictions and targets, and optional losses (if `loss_func`) using `dl`, max batches `n_batch`."     37     res = [torch.cat(o).cpu() for o in---> 38            zip(*validate(model, dl, cb_handler=cb_handler, pbar=pbar, average=False, n_batch=n_batch))]        global zip = undefined        global validate =         model = TabularModel(  (embeds): ModuleList(    (0): Embedding(1116, 81)    (1): Embedding(8, 5)    (2): Embedding(4, 3)    (3): Embedding(13, 7)    (4): Embedding(32, 11)    (5): Embedding(3, 3)    (6): Embedding(26, 10)    (7): Embedding(27, 10)    (8): Embedding(5, 4)    (9): Embedding(4, 3)    (10): Embedding(4, 3)    (11): Embedding(24, 9)    (12): Embedding(9, 5)    (13): Embedding(13, 7)    (14): Embedding(53, 15)    (15): Embedding(22, 9)    (16): Embedding(7, 5)    (17): Embedding(7, 5)    (18): Embedding(4, 3)    (19): Embedding(4, 3)    (20): Embedding(9, 5)    (21): Embedding(9, 5)    (22): Embedding(3, 3)    (23): Embedding(3, 3)  )  (emb_drop): Dropout(p=0.04)  (bn_cont): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)  (layers): Sequential(    (0): Linear(in_features=233, out_features=1000, bias=True)    (1): ReLU(inplace)    (2): BatchNorm1d(1000, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)    (3): Dropout(p=0.001)    (4): Linear(in_features=1000, out_features=500, bias=True)    (5): ReLU(inplace)    (6): BatchNorm1d(500, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)    (7): Dropout(p=0.01)    (8): Linear(in_features=500, out_features=1, bias=True)  ))        dl = None        cb_handler = CallbackHandler(callbacks=[], metrics=[], beta=0.98)        pbar = None        global average = undefined        n_batch = None     39     if loss_func is not None: res.append(calc_loss(res[0], res[1], loss_func))     40     if activ is not None: res[0] = activ(res[0])/mnt/963GB/Data/Python/Courses/fastai/fastai/fastai/basic_train.py in validate(model=TabularModel(  (embeds): ModuleList(    (0): E...in_features=500, out_features=1, bias=True)  )), dl=None, loss_func=None, cb_handler=CallbackHandler(callbacks=[], metrics=[], beta=0.98), pbar=None, average=False, n_batch=None)     47     with torch.no_grad():     48         val_losses,nums = [],[]---> 49         for xb,yb in progress_bar(dl, parent=pbar, leave=(pbar is not None)):        xb = undefined        yb = undefined        global progress_bar =         dl = None        global parent = undefined        pbar = None        global leave = undefined     50             if cb_handler: xb, yb = cb_handler.on_batch_begin(xb, yb, train=False)     51             val_losses.append(loss_batch(model, xb, yb, loss_func, cb_handler=cb_handler))~/miniconda3/envs/fastai-py3.7/lib/python3.7/site-packages/fastprogress/fastprogress.py in __init__(self=, gen=None, total=None, display=True, leave=False, parent=None, auto_update=True)    143 class NBProgressBar(ProgressBar):    144     def __init__(self, gen, total=None, display=True, leave=True, parent=None, auto_update=True):--> 145         self.progress = html_progress_bar(0, len(gen) if total is None else total, "")        self.progress = undefined        global html_progress_bar =         global len = undefined        gen = None        total = None    146         super().__init__(gen, total, display, leave, parent, auto_update)    147 TypeError: object of type 'NoneType' has no len()ipdb> gen.shape*** AttributeError: 'NoneType' object has no attribute 'shape'  Test N using: data = (TabularList.from_df(df, path=path, cat_names=cat_vars, cont_names=cont_vars, procs=procs, test_df=test_df)                   .split_by_idx(valid_idx)                   .label_from_df(cols=dep_var, label_cls=FloatList, log=True)                   .databunch())    ---------------------------------------------------------------------------TypeError                                 Traceback (most recent call last) in ----> 1 preds, y = learn.get_preds(DatasetType.Test)/mnt/963GB/Data/Python/Courses/fastai/fastai/fastai/basic_train.py in get_preds(self, ds_type, with_loss, n_batch, pbar)    228         lf = self.loss_func if with_loss else None    229         return get_preds(self.model, self.dl(ds_type), cb_handler=CallbackHandler(self.callbacks),--> 230                          activ=_loss_func2activ(self.loss_func), loss_func=lf, n_batch=n_batch, pbar=pbar)    231     232     def pred_batch(self, ds_type:DatasetType=DatasetType.Valid, batch:Tuple=None, reconstruct:bool=False) -> List[Tensor]:/mnt/963GB/Data/Python/Courses/fastai/fastai/fastai/basic_train.py in get_preds(model, dl, pbar, cb_handler, activ, loss_func, n_batch)     36     "Tuple of predictions and targets, and optional losses (if `loss_func`) using `dl`, max batches `n_batch`."     37     res = [torch.cat(o).cpu() for o in---> 38            zip(*validate(model, dl, cb_handler=cb_handler, pbar=pbar, average=False, n_batch=n_batch))]     39     if loss_func is not None: res.append(calc_loss(res[0], res[1], loss_func))     40     if activ is not None: res[0] = activ(res[0])/mnt/963GB/Data/Python/Courses/fastai/fastai/fastai/basic_train.py in validate(model, dl, loss_func, cb_handler, pbar, average, n_batch)     47     with torch.no_grad():     48         val_losses,nums = [],[]---> 49         for xb,yb in progress_bar(dl, parent=pbar, leave=(pbar is not None)):     50             if cb_handler: xb, yb = cb_handler.on_batch_begin(xb, yb, train=False)     51             val_losses.append(loss_batch(model, xb, yb, loss_func, cb_handler=cb_handler))~/miniconda3/envs/fastai-py3.7/lib/python3.7/site-packages/fastprogress/fastprogress.py in __init__(self, gen, total, display, leave, parent, auto_update)    143 class NBProgressBar(ProgressBar):    144     def __init__(self, gen, total=None, display=True, leave=True, parent=None, auto_update=True):--> 145         self.progress = html_progress_bar(0, len(gen) if total is None else total, "")    146         super().__init__(gen, total, display, leave, parent, auto_update)    147 TypeError: object of type 'NoneType' has no len()<jupyter_code>%debug
probs = np.exp(preds)
#get classes
d = {}
p = {}
i=0
for indx, prob in zip(indexes, probs):
    if i==0:
        print(f'prob: {prob}, dim: {prob.shape}, indx: {indx}')
    max_idx = np.argmax(prob)
    max_val = prob[max_idx].item()
    p[indx] = max_val
    prob_c = classes[max_idx]
    d[indx] = prob_c
len(d)
def pred_serial():
    preds = {}
    for idx, row in test_df.iterrows():
        pred = learn.predict(row)
        pred_str = pred[0].__str__()
        preds[idx]=pred_str
        if int(idx) % 10000 == 0:
            print(f'pred: {pred}, idx: {idx}')
    return preds
preds_d = pred_serial()<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup