<jupyter_start><jupyter_text>def my_loss(y_pred,y_true,weights):    if len(y_pred.shape)==len(y_true.shape):        window=(y_true>=0).to(torch.float)        loss = (F.binary_cross_entropy_with_logits(y_pred,y_true,reduction='none')*window*weights.expand_as(y_true)).mean()/(window.mean()+1e-7)    else:        window0=(y_true[...,0]>=0).to(torch.float)        window1=(y_true[...,1]>=0).to(torch.float)        loss0 = F.binary_cross_entropy_with_logits(y_pred,y_true[...,0],reduction='none')*window0*weights.expand_as(y_true[...,0])/(window0.mean()+1e-7)        loss1 = F.binary_cross_entropy_with_logits(y_pred,y_true[...,1],reduction='none')*window1*weights.expand_as(y_true[...,1])/(window1.mean()+1e-7)        loss = (y_true[...,2]*loss0+(1.0-y_true[...,2])*loss1).mean()     return loss<jupyter_code>class Metric():
    def __init__(self,weights,k=0.03):
        self.weights=weights
        self.k=k
        self.zero()
        
    def zero(self):
        self.loss_sum=0.
        self.loss_count=0.
        self.lossf=0.
        
    def calc(self,y_pred,y_true,prefix=""):
        window=(y_true>=0).to(torch.float)
        loss = (F.binary_cross_entropy_with_logits(y_pred,y_true,reduction='none')*window*self.weights.expand_as(y_true)).mean()/(window.mean()+1e-5)
        self.lossf=self.lossf*(1-self.k)+loss*self.k
        self.loss_sum=self.loss_sum+loss*window.sum()
        self.loss_count=self.loss_count+window.sum()
        return({prefix+'mloss':self.lossf})    
        
    def calc_sums(self,prefix=""):
        return({prefix+'mloss_tot':self.loss_sum/self.loss_count})    


#features=(features-features.mean())/features.std()
class SimpleModel(nn.Module):
    def __init__(self,in_size):
        super(SimpleModel, self).__init__()
        self.dont_do_grad=[]
        self.conv2d1=torch.nn.Conv2d(1, 128, (7,in_size), padding=(3,0))
        self.bn0=torch.nn.BatchNorm1d(128)
        self.relu0=torch.nn.ReLU()
        self.conv1d1=torch.nn.Conv1d(128, 128, 5, padding=2)
        self.bn1=torch.nn.BatchNorm1d(128)
        self.relu1=torch.nn.ReLU()
        self.conv1d2=torch.nn.Conv1d(128, 64, 3, padding=1)
        self.bn2=torch.nn.BatchNorm1d(64)
        self.relu2=torch.nn.ReLU()
        self.conv1d3=torch.nn.Conv1d(64, 6, 3, padding=1)
        
        
    def forward(self, x):
        x = self.conv2d1(x.unsqueeze(1)).squeeze(-1)
        x = self.bn0(x)
        x = self.relu0(x)
        x = self.conv1d1(x)
        x = self.bn1(x)
        x = self.relu1(x)
        x = self.conv1d2(x)
        x = self.bn2(x)
        x = self.relu2(x)
        out = self.conv1d3(x).transpose(-1,-2)
        return out 
                    
    def no_grad(self):
        for param in self.parameters():
            param.requires_grad=False

    def do_grad(self):
        for n,p in self.named_parameters():
            p.requires_grad=  not any(nd in n for nd in self.dont_do_grad)

class SimpleModel2(nn.Module):
    def __init__(self,in_size):
        super(SimpleModel2, self).__init__()
        self.dont_do_grad=[]
        self.conv2d1=torch.nn.Conv2d(1, 128, (9,in_size), padding=(4,0))
        self.bn0=torch.nn.BatchNorm1d(128)
        self.relu0=torch.nn.ReLU()
        self.conv1d1=torch.nn.Conv1d(128, 128, 7, padding=3)
        self.bn1=torch.nn.BatchNorm1d(128)
        self.relu1=torch.nn.ReLU()
        self.conv1d2=torch.nn.Conv1d(128, 64, 5, padding=2)
        self.bn2=torch.nn.BatchNorm1d(64)
        self.relu2=torch.nn.ReLU()
        self.conv1d3=torch.nn.Conv1d(64, 6, 3, padding=1)
        
        
    def forward(self, x):
        x = self.conv2d1(x.unsqueeze(1)).squeeze(-1)
        x = self.bn0(x)
        x = self.relu0(x)
        x = self.conv1d1(x)
        x = self.bn1(x)
        x = self.relu1(x)
        x = self.conv1d2(x)
        x = self.bn2(x)
        x = self.relu2(x)
        out = self.conv1d3(x).transpose(-1,-2)
        return out 
                    
    def no_grad(self):
        for param in self.parameters():
            param.requires_grad=False

    def do_grad(self):
        for n,p in self.named_parameters():
            p.requires_grad=  not any(nd in n for nd in self.dont_do_grad)

class ClassModel(nn.Module):
    def __init__(self,in_size):
        super(ClassModel, self).__init__()
        self.dont_do_grad=[]
        self.conv2d1=torch.nn.Conv2d(1, 128, (9,in_size), padding=(4,0))
        self.bn0=torch.nn.BatchNorm1d(128)
        self.relu0=torch.nn.ReLU()
        self.conv1d1=torch.nn.Conv1d(128, 128, 7, padding=3)
        self.bn1=torch.nn.BatchNorm1d(128)
        self.relu1=torch.nn.ReLU()
        self.conv1d2=torch.nn.Conv1d(128, 64, 5, padding=2)
        self.bn2=torch.nn.BatchNorm1d(64)
        self.relu2=torch.nn.ReLU()
        self.conv1d3=torch.nn.Conv1d(128, 6, 3, padding=1)
        
        self.conv2d1class=torch.nn.Conv2d(1, 128, (9,in_size), padding=(4,0))
        self.bn0class=torch.nn.BatchNorm1d(128)
        self.maxpool1class=torch.nn.MaxPool1d(3)
        self.conv1d1class=torch.nn.Conv1d(128, 128, 3, padding=1)
        self.bn1class=torch.nn.BatchNorm1d(128)
        self.maxpool2class=torch.nn.MaxPool1d(3)
        self.conv1d2class=torch.nn.Conv1d(128, 64, 2, padding=1)
        self.bn2class=torch.nn.BatchNorm1d(64)

        
        
    def forward(self, x):
        z=x
        x = self.conv2d1(x.unsqueeze(1)).squeeze(-1)
        x = self.bn0(x)
        x = self.relu0(x)
        x = self.conv1d1(x)
        x = self.bn1(x)
        x = self.relu1(x)
        x = self.conv1d2(x)
        x = self.bn2(x)
        x = self.relu2(x)
        z=self.conv2d1class(z.unsqueeze(1)).squeeze(-1)
        z=self.bn0class(z)
        z=self.maxpool1class(z)
        z=self.conv1d1class(z)
        z=self.maxpool2class(z)
        z=self.conv1d2class(z)
        z=self.bn2class(z)
        z=F.max_pool1d(z,kernel_size=z.shape[-1])
        z=z.expand_as(x)
        x=torch.cat([x,z],1)
        out = self.conv1d3(x).transpose(-1,-2)
        return out 
                    
    def no_grad(self):
        for param in self.parameters():
            param.requires_grad=False

    def do_grad(self):
        for n,p in self.named_parameters():
            p.requires_grad=  not any(nd in n for nd in self.dont_do_grad)

class ResModel(nn.Module):
    def __init__(self,in_size):
        super(ResModel, self).__init__()
        self.dont_do_grad=[]
        self.conv2d1=torch.nn.Conv2d(1, 64, (9,in_size), padding=(4,0))
        self.bn0=torch.nn.BatchNorm1d(64)
        self.relu0=torch.nn.ReLU()
        self.conv1d1=torch.nn.Conv1d(64, 64, 7, padding=3)
        self.bn1=torch.nn.BatchNorm1d(64)
        self.relu1=torch.nn.ReLU()
        self.conv1d2=torch.nn.Conv1d(128, 64, 5, padding=2)
        self.bn2=torch.nn.BatchNorm1d(64)
        self.relu2=torch.nn.ReLU()
        self.conv1d3=torch.nn.Conv1d(192, 6, 3, padding=1)
        
        
    def forward(self, x):
        x=x.unsqueeze(1)
        x = self.conv2d1(x).squeeze(-1)
        x = self.bn0(x)
        x0 = self.relu0(x)
        x = self.conv1d1(x0)
        x = self.bn1(x)
        x1 = self.relu1(x)
        x = torch.cat([x0,x1],1)
        x = self.conv1d2(x)
        x = self.bn2(x)
        x2 = self.relu2(x)
        x = torch.cat([x0,x1,x2],1)
        out = self.conv1d3(x).transpose(-1,-2)
        return out 
                    
    def no_grad(self):
        for param in self.parameters():
            param.requires_grad=False

    def do_grad(self):
        for n,p in self.named_parameters():
            p.requires_grad=  not any(nd in n for nd in self.dont_do_grad)

class ResModelPool(nn.Module):
    def __init__(self,in_size):
        super(ResModelPool, self).__init__()
        self.dont_do_grad=[]
        self.conv2d1=torch.nn.Conv2d(1, 64, (9,in_size),stride=(1,in_size), padding=(4,0))
        self.bn0=torch.nn.BatchNorm1d(64)
#        self.relu0=torch.nn.ReLU()
        self.conv1d1=torch.nn.Conv1d(64, 64, 7, padding=3)
        self.bn1=torch.nn.BatchNorm1d(64)
        self.relu1=torch.nn.ReLU()
        self.conv1d2=torch.nn.Conv1d(128, 64, 5, padding=2)
        self.bn2=torch.nn.BatchNorm1d(64)
        self.relu2=torch.nn.ReLU()
        self.conv1d3=torch.nn.Conv1d(192, 6, 3, padding=1)
        
        
    def forward(self, x):
        x=x.unsqueeze(1)
        x = self.conv2d1(x)
        x=F.max_pool2d(x,kernel_size=(1,x.shape[-1])).squeeze(-1)        
        x0 = self.bn0(x)
#        x0 = self.relu0(x)
        x = self.conv1d1(x0)
        x = self.bn1(x)
        x1 = self.relu1(x)
        x = torch.cat([x0,x1],1)
        x = self.conv1d2(x)
        x = self.bn2(x)
        x2 = self.relu2(x)
        x = torch.cat([x0,x1,x2],1)
        out = self.conv1d3(x).transpose(-1,-2)
        return out 
                    
    def no_grad(self):
        for param in self.parameters():
            param.requires_grad=False

    def do_grad(self):
        for n,p in self.named_parameters():
            p.requires_grad=  not any(nd in n for nd in self.dont_do_grad)

class ResDropModel(nn.Module):
    def __init__(self,in_size,dropout=0.2):
        super(ResDropModel, self).__init__()
        self.dont_do_grad=[]
        self.conv2d1=torch.nn.Conv2d(1, 64, (9,in_size), padding=(4,0))
        self.bn0=torch.nn.BatchNorm1d(64)
        self.relu0=torch.nn.ReLU()
        self.conv1d1=torch.nn.Conv1d(64, 64, 7, padding=3)
        self.bn1=torch.nn.BatchNorm1d(64)
        self.relu1=torch.nn.ReLU()
        self.conv1d2=torch.nn.Conv1d(128, 64, 5, padding=2)
        self.bn2=torch.nn.BatchNorm1d(64)
        self.relu2=torch.nn.ReLU()
        self.conv1d3=torch.nn.Conv1d(192, 6, 3, padding=1)
        self.dropout=dropout
        
    def forward(self, x):
        x = self.conv2d1(x.unsqueeze(1)).squeeze(-1)
        x = self.bn0(x)
        x = F.dropout(x,self.dropout)
        x0 = self.relu0(x) 
        x = self.conv1d1(x0)
#        x = self.bn1(x)
        x = F.dropout(x,self.dropout)
        x1 = self.relu1(x)
        x = torch.cat([x0,x1],1)
        x = self.conv1d2(x)
#        x = self.bn2(x)
        x = F.dropout(x,self.dropout)
        x2 = self.relu2(x)
        x = torch.cat([x0,x1,x2],1)
        x = F.dropout(x,self.dropout)
        out = self.conv1d3(x).transpose(-1,-2)
        return out 
                    
    def no_grad(self):
        for param in self.parameters():
            param.requires_grad=False

    def do_grad(self):
        for n,p in self.named_parameters():
            p.requires_grad=  not any(nd in n for nd in self.dont_do_grad)

fn = partial(torch.clamp,min=0,max=1)
class GenReLU(nn.Module):
    def __init__(self,leak=0,add=0,clamp=None):
        super(GenReLU, self).__init__()
        self.leak,self.add=leak,add
        if isinstance(clamp,tuple):
            self.clamp = partial(torch.clamp,min=clamp[0],max=clamp[1])
        elif clamp:
            self.clamp = partial(torch.clamp,min=-clamp,max=clamp)
        else:
            self.clamp=None
            
        
    def forward(self,x):
        x = F.leaky_relu(x,self.leak)
        if self.add:
            x=x+self.add
        if self.clamp:
            x = self.clamp(x)
        return x
        
        

class ResModelIn(nn.Module):
    def __init__(self,in_size):
        super(ResModelIn, self).__init__()
        self.dont_do_grad=[]
        self.conv2d1=torch.nn.Conv2d(1, 64, (9,in_size), padding=(4,0))
        self.bn0=torch.nn.BatchNorm1d(64)
        self.relu0=torch.nn.ReLU()
        self.conv1d1=torch.nn.Conv1d(64, 64, 3, padding=1)
        self.bn1=torch.nn.BatchNorm1d(64)
        self.relu1=torch.nn.ReLU()
        self.conv1d2=torch.nn.Conv1d(128, 64, 3, padding=1)
        self.bn2=torch.nn.BatchNorm1d(64)
        self.relu2=torch.nn.ReLU()
        self.conv1d3=torch.nn.Conv1d(192, 6, 3, padding=1)

        
        
    def forward(self, x):
        x = x.unsqueeze(1)
        x = self.conv2d1(x).squeeze(-1)
        x = self.bn0(x)
        x0 = self.relu0(x)
        x = self.conv1d1(x0)
        x = self.bn1(x)
        x1 = self.relu1(x)
        x = torch.cat([x0,x1],1)
        x = self.conv1d2(x)
        x = self.bn2(x)
        x2 = self.relu2(x)
        x = torch.cat([x0,x1,x2],1)
        x = self.conv1d3(x)
        out = x.transpose(-1,-2)
        return out 
                    
    def no_grad(self):
        for param in self.parameters():
            param.requires_grad=False

    def do_grad(self):
        for n,p in self.named_parameters():
            p.requires_grad=  not any(nd in n for nd in self.dont_do_grad)


class ResModelInR(nn.Module):
    def __init__(self,in_size):
        super(ResModelInR, self).__init__()
        self.dont_do_grad=[]
        self.conv2d1=torch.nn.Conv2d(1, 64, (9,in_size), padding=(4,0))
        self.bn0=torch.nn.BatchNorm1d(64)
        self.relu0= GenReLU(leak=0.01,add=-0.3,clamp=(-1,3))
        self.conv1d1=torch.nn.Conv1d(64, 64, 3, padding=1)
        self.bn1=torch.nn.BatchNorm1d(64)
        self.relu1=GenReLU(leak=0.01,add=-0.3,clamp=(-1,3))
        self.conv1d2=torch.nn.Conv1d(128, 64, 3, padding=1)
        self.bn2=torch.nn.BatchNorm1d(64)
        self.relu2=GenReLU(leak=0,add=0,clamp=(0,2))
        self.conv1d3=torch.nn.Conv1d(192, 6, 3, padding=1)

        
        
    def forward(self, x):
        x = x.unsqueeze(1)
        x = self.conv2d1(x).squeeze(-1)
        x = self.bn0(x)
        x0 = self.relu0(x)
        x = self.conv1d1(x0)
        x = self.bn1(x)
        x1 = self.relu1(x)
        x = torch.cat([x0,x1],1)
        x = self.conv1d2(x)
        x = self.bn2(x)
        x2 = self.relu2(x)
        x = torch.cat([x0,x1,x2],1)
        x = self.conv1d3(x)
        out = x.transpose(-1,-2)
        return out 
                    
    def no_grad(self):
        for param in self.parameters():
            param.requires_grad=False

    def do_grad(self):
        for n,p in self.named_parameters():
            p.requires_grad=  not any(nd in n for nd in self.dont_do_grad)

class BaseModel(nn.Module):
    def __init__(self):
        super(BaseModel, self).__init__()
        self.dont_do_grad=[]
        self.conv2d1=torch.nn.Conv2d(1, 128, (5,2208), padding=(2,0))
        self.relu0=torch.nn.ReLU()
        self.conv1d1=torch.nn.Conv1d(128, 6, 1)

        
        
    def forward(self, x):
        x = self.conv2d1(x.unsqueeze(1)).squeeze(-1)
        x = self.relu0(x)
        out = self.conv1d1(x).transpose(-1,-2)
        return out 
                    
    def no_grad(self):
        for param in self.parameters():
            param.requires_grad=False

    def do_grad(self):
        for n,p in self.named_parameters():
            p.requires_grad=  not any(nd in n for nd in self.dont_do_grad)

%matplotlib nbagg
for num_split in range(3):
    multi=3
    model_name,version = 'se_resnet101' , 'classifier_splits'
    print (model_name,version,num_split)
    pickle_file=open(outputs_dir+outputs_format.format(model_name,version,'features_train_tta',num_split),'rb')
    features=pickle.load(pickle_file)
    pickle_file.close()
    features.shape

    features=features.reshape(features.shape[0]//4,4,-1)
    features.shape
    split_train = train_df[train_df.PID.isin(set(split_sid[splits[num_split][0]]))].SeriesI.unique()
    split_validate =  train_df[train_df.PID.isin(set(split_sid[splits[num_split][1]]))].SeriesI.unique()

    np.random.seed(SEED+num_split)
    torch.manual_seed(SEED+num_split)
    torch.cuda.manual_seed(SEED+num_split)
    torch.backends.cudnn.deterministic = True
    batch_size=16
    num_workers=18
    num_epochs=24
    klr=1
    weights = torch.tensor([1.,1.,1.,1.,1.,2.],device=device)
    train_dataset=FullHeadDataset(train_df,
                                  split_train,
                                  features,
                                  'SeriesI',
                                  'ImagePositionZ',
                                  hemorrhage_types,
                                  multi=multi)                
    validate_dataset=FullHeadDataset(train_df,
                                     split_validate,
                                     torch.cat([features[:,i,:] for i in range(4)],-1),
                                     'SeriesI',
                                     'ImagePositionZ',
                                     hemorrhage_types)                

    model=ResModelPool(features.shape[-1])
    version=version+'_fullhead_resmodel_pool2_{}'.format(multi)
    _=model.to(device)
    #mixup=Mixup(device=device)
    loss_func=my_loss
    #fig,ax = plt.subplots(figsize=(10,7))
    #gr=loss_graph(fig,ax,num_epochs,len(train_dataset)//batch_size+1,limits=[0.02,0.06])
    num_train_optimization_steps = num_epochs*(len(train_dataset)//batch_size+int(len(train_dataset)%batch_size>0))
    sched=WarmupExpCosineWithWarmupRestartsSchedule( t_total=num_train_optimization_steps, cycles=2,tau=1)
    optimizer = BertAdam(model.parameters(),lr=klr*1e-3,schedule=sched)
    history,best_model= model_train(model,
                                    optimizer,
                                    train_dataset,
                                    batch_size,
                                    num_epochs,
                                    loss_func,
                                    weights=weights,
                                    do_apex=False,
                                    validate_dataset=validate_dataset,
                                    param_schedualer=None,
                                    weights_data=None,
                                    metric=Metric(torch.tensor([1.,1.,1.,1.,1.,2.])),
                                    return_model=True,
                                    best_average=3,
                                    num_workers=num_workers,
                                    sampler=None,
                                    graph=None)
    torch.save(best_model.state_dict(), models_dir+models_format.format(model_name,version,num_split))
%matplotlib nbagg
for num_split in range(3):
    multi=3
    model_name,version = 'se_resnext101_32x4d' , 'classifier_splits'
    print (model_name,version,num_split)
    pickle_file=open(outputs_dir+outputs_format.format(model_name,version,'features_train_tta',num_split),'rb')
    features=pickle.load(pickle_file)
    pickle_file.close()
    features.shape

    features=features.reshape(features.shape[0]//4,4,-1)
    features.shape
    split_train = train_df[train_df.PID.isin(set(split_sid[splits[num_split][0]]))].SeriesI.unique()
    split_validate =  train_df[train_df.PID.isin(set(split_sid[splits[num_split][1]]))].SeriesI.unique()

    np.random.seed(SEED+num_split)
    torch.manual_seed(SEED+num_split)
    torch.cuda.manual_seed(SEED+num_split)
    torch.backends.cudnn.deterministic = True
    batch_size=16
    num_workers=18
    num_epochs=24
    klr=1
    weights = torch.tensor([1.,1.,1.,1.,1.,2.],device=device)
    train_dataset=FullHeadDataset(train_df,
                                  split_train,
                                  features,
                                  'SeriesI',
                                  'ImagePositionZ',
                                  hemorrhage_types,
                                  multi=multi)                
    validate_dataset=FullHeadDataset(train_df,
                                     split_validate,
                                     torch.cat([features[:,i,:] for i in range(4)],-1),
                                     'SeriesI',
                                     'ImagePositionZ',
                                     hemorrhage_types)                

    model=ResModelPool(features.shape[-1])
    version=version+'_fullhead_resmodel_pool2_{}'.format(multi)
    _=model.to(device)
    #mixup=Mixup(device=device)
    loss_func=my_loss
    #fig,ax = plt.subplots(figsize=(10,7))
    #gr=loss_graph(fig,ax,num_epochs,len(train_dataset)//batch_size+1,limits=[0.02,0.06])
    num_train_optimization_steps = num_epochs*(len(train_dataset)//batch_size+int(len(train_dataset)%batch_size>0))
    sched=WarmupExpCosineWithWarmupRestartsSchedule( t_total=num_train_optimization_steps, cycles=2,tau=1)
    optimizer = BertAdam(model.parameters(),lr=klr*1e-3,schedule=sched)
    history,best_model= model_train(model,
                                    optimizer,
                                    train_dataset,
                                    batch_size,
                                    num_epochs,
                                    loss_func,
                                    weights=weights,
                                    do_apex=False,
                                    validate_dataset=validate_dataset,
                                    param_schedualer=None,
                                    weights_data=None,
                                    metric=Metric(torch.tensor([1.,1.,1.,1.,1.,2.])),
                                    return_model=True,
                                    best_average=3,
                                    num_workers=num_workers,
                                    sampler=None,
                                    graph=None)
    torch.save(best_model.state_dict(), models_dir+models_format.format(model_name,version,num_split))
%matplotlib nbagg
for num_split in range(3):
    multi=3
    model_name,version = 'Densenet161_3' , 'classifier_splits'
    print (model_name,version,num_split)
    pickle_file=open(outputs_dir+outputs_format.format(model_name,version,'features_train_tta2',num_split),'rb')
    features=pickle.load(pickle_file)
    pickle_file.close()
    features.shape

    features=features.reshape(features.shape[0]//4,4,-1)
    features.shape
    split_train = train_df[train_df.PID.isin(set(split_sid[splits[num_split][0]]))].SeriesI.unique()
    split_validate =  train_df[train_df.PID.isin(set(split_sid[splits[num_split][1]]))].SeriesI.unique()

    np.random.seed(SEED+num_split)
    torch.manual_seed(SEED+num_split)
    torch.cuda.manual_seed(SEED+num_split)
    torch.backends.cudnn.deterministic = True
    batch_size=16
    num_workers=18
    num_epochs=24
    klr=1
    weights = torch.tensor([1.,1.,1.,1.,1.,2.],device=device)
    train_dataset=FullHeadDataset(train_df,
                                  split_train,
                                  features,
                                  'SeriesI',
                                  'ImagePositionZ',
                                  hemorrhage_types,
                                  multi=multi)                
    validate_dataset=FullHeadDataset(train_df,
                                     split_validate,
                                     torch.cat([features[:,i,:] for i in range(4)],-1),
                                     'SeriesI',
                                     'ImagePositionZ',
                                     hemorrhage_types)                

    model=ResModelPool(features.shape[-1])
    version=version+'_fullhead_resmodel_pool2_{}'.format(multi)
    _=model.to(device)
    #mixup=Mixup(device=device)
    loss_func=my_loss
    #fig,ax = plt.subplots(figsize=(10,7))
    #gr=loss_graph(fig,ax,num_epochs,len(train_dataset)//batch_size+1,limits=[0.02,0.06])
    num_train_optimization_steps = num_epochs*(len(train_dataset)//batch_size+int(len(train_dataset)%batch_size>0))
    sched=WarmupExpCosineWithWarmupRestartsSchedule( t_total=num_train_optimization_steps, cycles=2,tau=1)
    optimizer = BertAdam(model.parameters(),lr=klr*1e-3,schedule=sched)
    history,best_model= model_train(model,
                                    optimizer,
                                    train_dataset,
                                    batch_size,
                                    num_epochs,
                                    loss_func,
                                    weights=weights,
                                    do_apex=False,
                                    validate_dataset=validate_dataset,
                                    param_schedualer=None,
                                    weights_data=None,
                                    metric=Metric(torch.tensor([1.,1.,1.,1.,1.,2.])),
                                    return_model=True,
                                    best_average=3,
                                    num_workers=num_workers,
                                    sampler=None,
                                    graph=None)
    torch.save(best_model.state_dict(), models_dir+models_format.format(model_name,version,num_split))
%matplotlib nbagg
for num_split in range(3):
    multi=3
    model_name,version = 'Densenet169_3' , 'classifier_splits'
    print (model_name,version,num_split)
    pickle_file=open(outputs_dir+outputs_format.format(model_name,version,'features_train_tta2',num_split),'rb')
    features=pickle.load(pickle_file)
    pickle_file.close()
    features.shape

    features=features.reshape(features.shape[0]//4,4,-1)
    features.shape
    split_train = train_df[train_df.PID.isin(set(split_sid[splits[num_split][0]]))].SeriesI.unique()
    split_validate =  train_df[train_df.PID.isin(set(split_sid[splits[num_split][1]]))].SeriesI.unique()

    np.random.seed(SEED+num_split)
    torch.manual_seed(SEED+num_split)
    torch.cuda.manual_seed(SEED+num_split)
    torch.backends.cudnn.deterministic = True
    batch_size=16
    num_workers=18
    num_epochs=24
    klr=1
    weights = torch.tensor([1.,1.,1.,1.,1.,2.],device=device)
    train_dataset=FullHeadDataset(train_df,
                                  split_train,
                                  features,
                                  'SeriesI',
                                  'ImagePositionZ',
                                  hemorrhage_types,
                                  multi=multi)                
    validate_dataset=FullHeadDataset(train_df,
                                     split_validate,
                                     torch.cat([features[:,i,:] for i in range(4)],-1),
                                     'SeriesI',
                                     'ImagePositionZ',
                                     hemorrhage_types)                

    model=ResModelPool(features.shape[-1])
    version=version+'_fullhead_resmodel_pool2_{}'.format(multi)
    _=model.to(device)
    #mixup=Mixup(device=device)
    loss_func=my_loss
    #fig,ax = plt.subplots(figsize=(10,7))
    #gr=loss_graph(fig,ax,num_epochs,len(train_dataset)//batch_size+1,limits=[0.02,0.06])
    num_train_optimization_steps = num_epochs*(len(train_dataset)//batch_size+int(len(train_dataset)%batch_size>0))
    sched=WarmupExpCosineWithWarmupRestartsSchedule( t_total=num_train_optimization_steps, cycles=2,tau=1)
    optimizer = BertAdam(model.parameters(),lr=klr*1e-3,schedule=sched)
    history,best_model= model_train(model,
                                    optimizer,
                                    train_dataset,
                                    batch_size,
                                    num_epochs,
                                    loss_func,
                                    weights=weights,
                                    do_apex=False,
                                    validate_dataset=validate_dataset,
                                    param_schedualer=None,
                                    weights_data=None,
                                    metric=Metric(torch.tensor([1.,1.,1.,1.,1.,2.])),
                                    return_model=True,
                                    best_average=3,
                                    num_workers=num_workers,
                                    sampler=None,
                                    graph=None)
    torch.save(best_model.state_dict(), models_dir+models_format.format(model_name,version,num_split))
%matplotlib nbagg
for num_split in range(3):
    multi=3
    model_name,version = 'Densenet201_3' , 'classifier_splits'
    print (model_name,version,num_split)
    pickle_file=open(outputs_dir+outputs_format.format(model_name,version,'features_train_tta',num_split),'rb')
    features=pickle.load(pickle_file)
    pickle_file.close()
    features.shape

    features=features.reshape(features.shape[0]//4,4,-1)
    features.shape
    split_train = train_df[train_df.PID.isin(set(split_sid[splits[num_split][0]]))].SeriesI.unique()
    split_validate =  train_df[train_df.PID.isin(set(split_sid[splits[num_split][1]]))].SeriesI.unique()

    np.random.seed(SEED+num_split)
    torch.manual_seed(SEED+num_split)
    torch.cuda.manual_seed(SEED+num_split)
    torch.backends.cudnn.deterministic = True
    batch_size=16
    num_workers=18
    num_epochs=24
    klr=1
    weights = torch.tensor([1.,1.,1.,1.,1.,2.],device=device)
    train_dataset=FullHeadDataset(train_df,
                                  split_train,
                                  features,
                                  'SeriesI',
                                  'ImagePositionZ',
                                  hemorrhage_types,
                                  multi=multi)                
    validate_dataset=FullHeadDataset(train_df,
                                     split_validate,
                                     torch.cat([features[:,i,:] for i in range(4)],-1),
                                     'SeriesI',
                                     'ImagePositionZ',
                                     hemorrhage_types)                

    model=ResModelPool(features.shape[-1])
    version=version+'_fullhead_resmodel_pool2_{}'.format(multi)
    _=model.to(device)
    #mixup=Mixup(device=device)
    loss_func=my_loss
    #fig,ax = plt.subplots(figsize=(10,7))
    #gr=loss_graph(fig,ax,num_epochs,len(train_dataset)//batch_size+1,limits=[0.02,0.06])
    num_train_optimization_steps = num_epochs*(len(train_dataset)//batch_size+int(len(train_dataset)%batch_size>0))
    sched=WarmupExpCosineWithWarmupRestartsSchedule( t_total=num_train_optimization_steps, cycles=2,tau=1)
    optimizer = BertAdam(model.parameters(),lr=klr*1e-3,schedule=sched)
    history,best_model= model_train(model,
                                    optimizer,
                                    train_dataset,
                                    batch_size,
                                    num_epochs,
                                    loss_func,
                                    weights=weights,
                                    do_apex=False,
                                    validate_dataset=validate_dataset,
                                    param_schedualer=None,
                                    weights_data=None,
                                    metric=Metric(torch.tensor([1.,1.,1.,1.,1.,2.])),
                                    return_model=True,
                                    best_average=3,
                                    num_workers=num_workers,
                                    sampler=None,
                                    graph=None)
    torch.save(best_model.state_dict(), models_dir+models_format.format(model_name,version,num_split))<jupyter_output>Densenet201_3 classifier_splits 0<jupyter_text>Mid Inference for sanity<jupyter_code>pred_list=[]
pred_list_tmp=[]
for num_split in tqdm_notebook(range(3)):
    model_name,version = 'Densenet169_3' , 'classifier_splits'
    pickle_file=open(outputs_dir+outputs_format.format(model_name,version,'features_test',num_split),'rb')
    features=pickle.load(pickle_file)
    pickle_file.close()
    features=features.reshape(features.shape[0]//8,8,-1)

    model=ResModelPool(features.shape[-1])
    version=version+'_fullhead_resmodel_pool2_3'

    model.load_state_dict(torch.load(models_dir+models_format.format(model_name,version,num_split),map_location=torch.device(device)))
    test_dataset=train_dataset=FullHeadDataset(test_df,
                                  test_df.SeriesI.unique(),
                                  features,
                                  'SeriesI',
                                  'ImagePositionZ',multi=4)
    for i in tqdm_notebook(range(32),leave=False):
        pred_list.append(torch.sigmoid(model_run(model,test_dataset,do_apex=False,batch_size=128))[...,None])
        pred_list_tmp.append(pred_list[-1])
pred169=torch.cat(pred_list_tmp,-1).mean(-1)
pred_list_tmp=[]
for num_split in tqdm_notebook(range(3)):
    model_name,version = 'Densenet161_3' , 'classifier_splits'
    pickle_file=open(outputs_dir+outputs_format.format(model_name,version,'features_test',num_split),'rb')
    features=pickle.load(pickle_file)
    pickle_file.close()
    features=features.reshape(features.shape[0]//8,8,-1)

    model=ResModelPool(features.shape[-1])
    version=version+'_fullhead_resmodel_pool2_3'

    model.load_state_dict(torch.load(models_dir+models_format.format(model_name,version,num_split),map_location=torch.device(device)))
    test_dataset=train_dataset=FullHeadDataset(test_df,
                                  test_df.SeriesI.unique(),
                                  features,
                                  'SeriesI',
                                  'ImagePositionZ',multi=4)
    for i in tqdm_notebook(range(32),leave=False):
        pred_list.append(torch.sigmoid(model_run(model,test_dataset,do_apex=False,batch_size=128))[...,None])
        pred_list_tmp.append(pred_list[-1])
pred161=torch.cat(pred_list_tmp,-1).mean(-1)
pred_list_tmp=[]
for num_split in tqdm_notebook(range(3)):
    model_name,version = 'Densenet201_3' , 'classifier_splits'
    pickle_file=open(outputs_dir+outputs_format.format(model_name,version,'features_test',num_split),'rb')
    features=pickle.load(pickle_file)
    pickle_file.close()
    features=features.reshape(features.shape[0]//8,8,-1)

    model=ResModelPool(features.shape[-1])
    version=version+'_fullhead_resmodel_pool2_3'

    model.load_state_dict(torch.load(models_dir+models_format.format(model_name,version,num_split),map_location=torch.device(device)))
    test_dataset=train_dataset=FullHeadDataset(test_df,
                                  test_df.SeriesI.unique(),
                                  features,
                                  'SeriesI',
                                  'ImagePositionZ',multi=4)
    for i in tqdm_notebook(range(32),leave=False):
        pred_list.append(torch.sigmoid(model_run(model,test_dataset,do_apex=False,batch_size=128))[...,None])
        pred_list_tmp.append(pred_list[-1])
pred201=torch.cat(pred_list_tmp,-1).mean(-1)
pred_list_tmp=[]
for num_split in tqdm_notebook(range(3)):
    model_name,version = 'se_resnext101_32x4d' , 'classifier_splits'
    pickle_file=open(outputs_dir+outputs_format.format(model_name,version,'features_test',num_split),'rb')
    features=pickle.load(pickle_file)
    pickle_file.close()
    features=features.reshape(features.shape[0]//8,8,-1)

    model=ResModelPool(features.shape[-1])
    version=version+'_fullhead_resmodel_pool2_3'

    model.load_state_dict(torch.load(models_dir+models_format.format(model_name,version,num_split),map_location=torch.device(device)))
    test_dataset=train_dataset=FullHeadDataset(test_df,
                                  test_df.SeriesI.unique(),
                                  features,
                                  'SeriesI',
                                  'ImagePositionZ',multi=4)
    for i in tqdm_notebook(range(32),leave=False):
        pred_list.append(torch.sigmoid(model_run(model,test_dataset,do_apex=False,batch_size=128))[...,None])
        pred_list_tmp.append(pred_list[-1])
predse=torch.cat(pred_list_tmp,-1).mean(-1)

pred_list_tmp=[]
for num_split in tqdm_notebook(range(5)):
    model_name,version = 'se_resnext101_32x4d' , 'new_splits'
    pickle_file=open(outputs_dir+outputs_format.format(model_name,version,'features_test',num_split),'rb')
    features=pickle.load(pickle_file)
    pickle_file.close()
    features=features.reshape(features.shape[0]//8,8,-1)

    model=ResModelPool(features.shape[-1])
    version=version+'_fullhead_resmodel_pool2_3'

    model.load_state_dict(torch.load(models_dir+models_format.format(model_name,version,num_split),map_location=torch.device(device)))
    test_dataset=train_dataset=FullHeadDataset(test_df,
                                  test_df.SeriesI.unique(),
                                  features,
                                  'SeriesI',
                                  'ImagePositionZ',multi=4)
    for i in tqdm_notebook(range(32),leave=False):
        pred_list.append(torch.sigmoid(model_run(model,test_dataset,do_apex=False,batch_size=128))[...,None])
        pred_list_tmp.append(pred_list[-1])
predse=torch.cat(pred_list_tmp,-1).mean(-1)
pred_list_tmp=[]
for num_split in tqdm_notebook(range(3)):
    model_name,version = 'se_resnet101' , 'classifier_splits'
    pickle_file=open(outputs_dir+outputs_format.format(model_name,version,'features_test',num_split),'rb')
    features=pickle.load(pickle_file)
    pickle_file.close()
    features=features.reshape(features.shape[0]//8,8,-1)

    model=ResModelPool(features.shape[-1])
    version=version+'_fullhead_resmodel_pool2_3'

    model.load_state_dict(torch.load(models_dir+models_format.format(model_name,version,num_split),map_location=torch.device(device)))
    test_dataset=train_dataset=FullHeadDataset(test_df,
                                  test_df.SeriesI.unique(),
                                  features,
                                  'SeriesI',
                                  'ImagePositionZ',multi=4)
    for i in tqdm_notebook(range(32),leave=False):
        pred_list.append(torch.sigmoid(model_run(model,test_dataset,do_apex=False,batch_size=128))[...,None])
        pred_list_tmp.append(pred_list[-1])
predseres=torch.cat(pred_list_tmp,-1).mean(-1)
pred=torch.cat(pred_list_tmp,-1).mean(-1)
pred.shape
len(pred_list)
pred_list[0].shape
pred=torch.cat(pred_list,-1).mean(-1)
images_id_list=[]
dummeys=[]
image_arr=test_df.PatientID.values
ref_arr=test_df.SeriesI.values
order_arr=test_df.ImagePositionZ.values
for s in tqdm_notebook(test_df.SeriesI.unique()):
    dumm=np.zeros(60)
    head_idx = np.where(ref_arr==s)[0]
    sorted_head_idx=head_idx[np.argsort(order_arr[head_idx])]
    images_id_list.append(image_arr[sorted_head_idx])
    dumm[0:head_idx.shape[0]]=1
    dummeys.append(dumm)
image_ids=np.concatenate(images_id_list)
preds=pred.reshape(pred.shape[0]*pred.shape[1],6).numpy()[np.concatenate(dummeys)==1]

image_ids.shape

preds.shape
len(pred_list)
images_id_list=[]
dummeys=[]
image_arr=test_df.PatientID.values
ref_arr=test_df.SeriesI.values
order_arr=test_df.ImagePositionZ.values
for s in tqdm_notebook(test_df.SeriesI.unique()):
    dumm=np.zeros(60)
    head_idx = np.where(ref_arr==s)[0]
    sorted_head_idx=head_idx[np.argsort(order_arr[head_idx])]
    images_id_list.append(image_arr[sorted_head_idx])
    dumm[0:head_idx.shape[0]]=1
    dummeys.append(dumm)
image_ids=np.concatenate(images_id_list)
preds=pred.reshape(pred.shape[0]*pred.shape[1],6).numpy()[np.concatenate(dummeys)==1]

image_ids.shape

preds.shape
submission_df=get_submission_ids(image_ids,torch.tensor(preds),do_sigmoid=False)
submission_df.head(12)
submission_df.shape
sub_num=51
submission_df.to_csv('/media/hd/notebooks/data/RSNA/submissions/submission{}.csv'.format(sub_num),
                                                                  index=False, columns=['ID','Label'])

#!/home/reina/anaconda3/bin/kaggle competitions submit rsna-intracranial-hemorrhage-detection -f /media/hd/notebooks/data/RSNA/submissions/submission51.csv -m "all models, se weight*2, with post pool_3 tta 64, mean after sigmoid"<jupyter_output>100%|███████████████████████████████████████| 16.9M/16.9M [01:52<00:00, 157kB/s]
100%|███████████████████████████████████████| 16.9M/16.9M [01:52<00:00, 157kB/s]
Successfully submitted to RSNA Intracranial Hemorrhage DetectionSuccessfully submitted to RSNA Intracranial Hemorrhage Detection
StarcoderdataJupyterStructuredCleanDedup