<jupyter_start><jupyter_text>1. Data Collection<jupyter_code>train = pd.read_csv('/kaggle/input/bengaliai-cv19/train.csv') 
test = pd.read_csv('/kaggle/input/bengaliai-cv19/test.csv')
class_map = pd.read_csv('/kaggle/input/bengaliai-cv19/class_map.csv')
sample_submission = pd.read_csv('/kaggle/input/bengaliai-cv19/sample_submission.csv')

# later do for all parquet files
train_0 = pd.read_parquet('/kaggle/input/bengaliai-cv19/train_image_data_0.parquet')
# train_1 = pd.read_parquet('/kaggle/input/bengaliai-cv19/train_image_data_1.parquet')
# train_2 = pd.read_parquet('/kaggle/input/bengaliai-cv19/train_image_data_2.parquet')
# train_3 = pd.read_parquet('/kaggle/input/bengaliai-cv19/train_image_data_3.parquet')

test_0 = pd.read_parquet('/kaggle/input/bengaliai-cv19/test_image_data_0.parquet')
test_1 = pd.read_parquet('/kaggle/input/bengaliai-cv19/test_image_data_1.parquet')
test_2 = pd.read_parquet('/kaggle/input/bengaliai-cv19/test_image_data_2.parquet')
test_3 = pd.read_parquet('/kaggle/input/bengaliai-cv19/test_image_data_3.parquet')
train.head()
test.head()
class_map.head()
train_0.head()
print('train shape', train.shape)
print('test shape', test.shape)
print('class_map shape', class_map.shape)
print('train_0 shape', train_0.shape)
print('test_0 shape', test_0.shape)
def display_image_from_pixels(data, subplots_size=5, smaller_data_alert=False):
    plt.figure()
    fig, ax = plt.subplots(subplots_size, subplots_size, figsize=(12,12))

    for i, index in enumerate(data.index):
        image_id = data.iloc[i]['image_id']
        image = data.iloc[i].drop('image_id').values.astype(np.uint8)
        image = Image.fromarray(image.reshape(137, 236))
        
        ax[i//subplots_size, i%subplots_size].imshow(image)
        ax[i//subplots_size, i%subplots_size].set_title(image_id)
        ax[i//subplots_size, i%subplots_size].axis('off')
        
    if smaller_data_alert:
        for empty_subplot in range(3, 25):
            ax[empty_subplot//subplots_size, empty_subplot%subplots_size].set_visible(False)
            

display_image_from_pixels(train_0.sample(25))
display_image_from_pixels(test_0, smaller_data_alert=True)<jupyter_output><empty_output><jupyter_text>2. Exploratory Data Analysis (EDA)  2.1. Show all possible class map signs<jupyter_code>def show_class_maps():
    print('-----------------')
    print('grapheme_root')
    print(class_map.loc[class_map['component_type'] == 'grapheme_root']['component'].values)
    print('-----------------')
    print('map_vowel')
    print(class_map.loc[class_map['component_type'] == 'vowel_diacritic']['component'].values)
    print('-----------------')
    print('map_diacritic')
    print(class_map.loc[class_map['component_type'] == 'consonant_diacritic']['component'].values)
    
show_class_maps()<jupyter_output><empty_output><jupyter_text>2.2. Get frequency of class map occurrences in train set<jupyter_code>def plot_freq(column='grapheme'):
    col = train[column].value_counts().rename_axis(column).reset_index(name='count')
    fig = px.bar(col, y='count', x=column)
    fig.show()
    
plot_freq('grapheme')
plot_freq('grapheme_root')
plot_freq('vowel_diacritic')
plot_freq('consonant_diacritic')<jupyter_output><empty_output><jupyter_text>2.3. Plot feature occurrence dependencies (encoded)<jupyter_code>def features_heatmap(feature1, feature2, width, length):
    df = train.groupby([feature1, feature2])['grapheme'].count().reset_index() 
    df = df.pivot(feature1, feature2, 'grapheme')
    plt.figure(figsize=(width, length))
    sns.heatmap(df, annot=True, fmt='3.0f', linewidths=.5, cmap='Blues')
    
features_heatmap('vowel_diacritic','consonant_diacritic' ,12 , 4)
#%%javascript
#IPython.OutputArea.auto_scroll_threshold = 9999;
features_heatmap('grapheme_root','consonant_diacritic', 12, 40)
features_heatmap('grapheme_root','vowel_diacritic', 18, 40)<jupyter_output><empty_output><jupyter_text>3. Feature Engineering (FE)<jupyter_code>class FE(object):
    __constraints__ = {'WIDTH': 137, 'HEIGHT': 236, 'END_SIZE': 128}

    @staticmethod
    def make_2d(dataset, vector):
        image = dataset.iloc[vector].drop('image_id').values.astype(np.uint8)
        image = image.reshape(FE.__constraints__['WIDTH'], FE.__constraints__['HEIGHT'])/1
        return image

    def crop_top(self, image, threshold):
        idx = 0
        for row in range(FE.__constraints__['WIDTH']):
            if np.sum(image[row]) / 255 > threshold:
                idx += 1
            else:
                return idx

    def crop_bot(self, image, threshold):
        idx = 0
        for row in reversed(range(FE.__constraints__['WIDTH'])):
            if np.sum(image[row]) / 255 > threshold:
                idx += 1
            else:
                return FE.__constraints__['WIDTH'] - idx    

    def crop_left(self, image, threshold):
        idx = 0
        for col in range(FE.__constraints__['HEIGHT']):
            if np.sum(image[:, col]) / 255 > threshold-95:
                idx += 1
            else:
                return idx

    def crop_right(self, image, threshold):
        idx = 0
        for col in reversed(range(FE.__constraints__['HEIGHT'])):
            if np.sum(image[:, col]) / 255 > threshold-95:
                idx += 1
            else:
                return FE.__constraints__['HEIGHT'] - idx 

    def crop_resize_image(self, image, threshold=230):
        return cv2.resize(image[self.crop_top(image, threshold): self.crop_bot(image, threshold), self.crop_left(image, threshold): self.crop_right(image, threshold)], (FE.__constraints__['END_SIZE'], FE.__constraints__['END_SIZE']), interpolation = INTER_AREA)

    @staticmethod
    def random_aug_mix(image, prob=0.5):
        image = cvtColor(image.astype(np.uint8), COLOR_GRAY2RGB)
        if prob > random.random():
            image = add_fog(image, fog_coef=0.5, alpha_coef=0, haze_list=[])
        if prob > random.random():
            image = add_snow(image, snow_point=1, brightness_coeff=0.2)
        if prob > random.random():
            image = elastic_transform(image, alpha=8, sigma=1, alpha_affine=0.8, interpolation=1, border_mode=4, value=None, random_state=None, approximate=False)
        if prob > random.random():
            image = iso_noise(image, color_shift=5, intensity=5)
        return np.moveaxis(image[:,:,:1], -1, 0)<jupyter_output><empty_output><jupyter_text>3.1. Cropping, Centering and Resizing images<jupyter_code>test = pd.concat([test_0, test_1, test_2, test_3])
# prepare datasets (change dims)
images_test = np.zeros(((137, 236, 12)))
for vector in tqdm(range(test.shape[0])):
    images_test[:,:,vector] = FE.make_2d(test, vector)
# dims after resize
resized_test = np.zeros(((128, 128, 12)))
for vector in tqdm(range(test.shape[0])): 
    resized_test[:,:,vector] = FE().crop_resize_image(images_test[:,:,vector], threshold=230)
del images_test<jupyter_output><empty_output><jupyter_text>3.3. AugMix train set calibration using albumentations<jupyter_code># for torch
X_test = np.zeros(((12, 1, 128, 128)))

# for keras
# X_train = np.zeros(((50210, 128, 128, 1)))
# X_test = np.zeros(((12, 128, 128, 1)))
for vector in tqdm(range(test.shape[0])):
    X_test[vector,:,:,:] = FE.random_aug_mix(resized_test[:,:,vector], prob=0)
del resized_test<jupyter_output><empty_output><jupyter_text>3.4. Merging previous steps into pipeline for future training sets preparation<jupyter_code>TRAIN_SIZE = 10000
# @cuda.autojit
def train_preprocessing_pipeline(train_set, train_step):
    print('{} train step processing'.format(train_step))
    images_train = np.zeros(((137, 236, TRAIN_SIZE)))
    
    for vector in tqdm(range(TRAIN_SIZE)): #range(train_0.shape[0])
        images_train[:,:,vector] = FE.make_2d(train_set, vector + (TRAIN_SIZE * train_step))
        
    resized_train = np.zeros(((128, 128, TRAIN_SIZE)))
    
    for vector in tqdm(range(TRAIN_SIZE)): #train_0.shape[0]
        resized_train[:,:,vector] = FE().crop_resize_image(images_train[:,:,vector], threshold=230)
        
    del images_train
    
    X_train = np.zeros(((TRAIN_SIZE, 1, 128, 128)))
    
    for vector in tqdm(range(TRAIN_SIZE)): 
        X_train[vector,:,:,:] = FE.random_aug_mix(resized_train[:,:,vector], prob=0.3)
    
    del resized_train
    
    print('processed')
    
    return data.DataLoader(
                        tensor(X_train),
                        batch_size=BATCH_SIZE,
                        num_workers=N_WORKERS,
                        shuffle=False
                    )<jupyter_output><empty_output><jupyter_text>4. Modeling<jupyter_code>device = "cuda:0" if torch.cuda.is_available() else "cpu"
device<jupyter_output><empty_output><jupyter_text>4.1. NN Architecture<jupyter_code>class ConvBlock(nn.Module):
    def __init__(self, in_channels, out_channels):        
        super(ConvBlock, self).__init__()
        
        self.conv2d1 = nn.Conv2d(in_channels=in_channels, out_channels=6, kernel_size=6, stride=2, padding=2)
        self.batch_norm = nn.BatchNorm2d(num_features=6)
        self.relu = nn.ReLU(True)
        self.max_pooling =  nn.MaxPool2d(kernel_size=2, stride=2, padding=0)
        self.conv2d2 = nn.Conv2d(in_channels=6, out_channels=out_channels, kernel_size=6, stride=2, padding=0)
        
    def forward(self, x):
        x = self.conv2d1(x)
        x = self.batch_norm(x)
        x = self.relu(x)
        x = self.max_pooling(x)
        x = self.conv2d2(x)
        return self.relu(x)
    
class ResidualBlock(nn.Module):
    def __init__(self): # prob
        super(ResidualBlock, self).__init__()
        self.id_block = nn.Sequential(
                            nn.Conv2d(in_channels=8, out_channels=8, kernel_size=3, stride=1, padding=1),
                            #nn.BatchNorm2d(num_features=8) 
                            nn.MaxPool2d(kernel_size=3, padding=1, stride=1)
                            )
        
        self.skip = nn.Sequential()
            
    def forward(self, x):
        residual = x
        x = self.id_block(x)
        x += self.skip(residual)
        return nn.ReLU(True)(x)


class BottleNeck(nn.Module):
    def __init__(self, in_channels, out_channels):
        super(BottleNeck, self).__init__()
        self.bottle_neck = nn.Conv2d(in_channels=in_channels, out_channels=out_channels, 
                                     kernel_size=1, stride=1, padding=0)

    def forward(self, x): 
        return self.bottle_neck(x)

class ResNet(nn.Module):
    def __init__(self):
        super(ResNet, self).__init__()
        
#         self.backbone = resnet34(pretrained=False)
        
        self.conv_block = ConvBlock(in_channels=1, out_channels=16)
        
        self.bottle_neck = BottleNeck(in_channels=16, out_channels=8)
        
        self.res_blocks = ResidualBlock()
        
        self.average_pooling = nn.AvgPool2d(kernel_size=2, stride=2, padding=0)
        
        self.flatten = nn.Flatten()
        self.norm = nn.BatchNorm2d(num_features=8)
        
        self.root = nn.Linear(392, 168)
        self.vowel = nn.Linear(392, 11)
        self.consonant = nn.Linear(392, 7)

    def forward(self, x):
        
       # x = self.forward_backbone(x)
        
        x = self.conv_block(x)
        x = self.bottle_neck(x)
        
        x = self.res_blocks(x)
        x = self.res_blocks(x)
        x = self.res_blocks(x)
        x = self.res_blocks(x)
        x = self.res_blocks(x)
        x = self.res_blocks(x)
        
        x = self.res_blocks(x)
        x = self.res_blocks(x)
        x = self.res_blocks(x)
        x = self.res_blocks(x)
        x = self.res_blocks(x)
        x = self.res_blocks(x)
        
        
        x = self.average_pooling(x)
        x = self.flatten(x)
        
        root = self.root(x)
        vowel = self.vowel(x)
        consonant = self.consonant(x)
        
        return root, vowel, consonant<jupyter_output><empty_output><jupyter_text>4.2. Constraints setting<jupyter_code>BATCH_SIZE = TRAIN_SIZE // 20 #32
N_WORKERS = 4
N_EPOCHS = 10
DEPLOYMENT_WEIGHTS = 'baseline_weights.pth'<jupyter_output><empty_output><jupyter_text>4.3. Model Template<jupyter_code>model = ResNet().to(device)
criterion = nn.BCEWithLogitsLoss() # KLDivLoss()
optimizer = Adam(model.parameters(), lr=0.5)
scheduler = ReduceLROnPlateau(optimizer, 'min', min_lr=0.1, verbose=True) # patience=1000, min_lr=0.1, 
epoch_losses = []
def train_nn(train_set, train_step, data_loader_train = []):

    for epoch in range(N_EPOCHS):

            print('Epoch {}/{}'.format(epoch, N_EPOCHS - 1))
            print('-' * 10)

            model.train()
            tr_loss = 0  
            
            if epoch == 0:
                data_loader_train = train_preprocessing_pipeline(train_set, train_step)
                
            for step, batch in enumerate(tqdm(data_loader_train)):
                inputs = tensor(batch)
                l_graph = tensor(Y_train_root[(batch.shape[0]*step):(batch.shape[0]*(step+1))])
                l_vowel = tensor(Y_train_vowel[(batch.shape[0]*step):(batch.shape[0]*(step+1))])
                l_conso = tensor(Y_train_consonant[(batch.shape[0]*step):(batch.shape[0]*(step+1))])

                inputs = inputs.to(device, dtype=torch.float)
                l_graph = l_graph.to(device, dtype=torch.float)
                l_vowel = l_vowel.to(device, dtype=torch.float)
                l_conso = l_conso.to(device, dtype=torch.float)

                out_graph, out_vowel, out_conso = model(inputs)

                loss_graph = criterion(out_graph, l_graph)
                loss_vowel = criterion(out_vowel, l_vowel)
                loss_conso = criterion(out_conso, l_conso)

                loss = loss_graph + loss_vowel + loss_conso

                scheduler.step(loss)
                loss.backward()

                tr_loss += loss.item()

                optimizer.step()
                optimizer.zero_grad()

            epoch_losses.append(tr_loss / len(data_loader_train))
            print('Training Loss: {:.4f}'.format(epoch_losses[-1]))
def make_labels(step):
    return tensor(pd.get_dummies(train['grapheme_root'][(TRAIN_SIZE * step):(TRAIN_SIZE * (step + 1))]).values),\
           tensor(pd.get_dummies(train['vowel_diacritic'][(TRAIN_SIZE * step):(TRAIN_SIZE * (step + 1))]).values),\
           tensor(pd.get_dummies(train['consonant_diacritic'][(TRAIN_SIZE * step):(TRAIN_SIZE * (step + 1))]).values)<jupyter_output><empty_output><jupyter_text>4.4. Training<jupyter_code>print('train_0 is being processed')
Y_train_root, Y_train_vowel, Y_train_consonant = make_labels(0)
train_nn(train_0, 0)

Y_train_root, Y_train_vowel, Y_train_consonant = make_labels(1)
train_nn(train_0, 1)

Y_train_root, Y_train_vowel, Y_train_consonant = make_labels(2)
train_nn(train_0, 2)

Y_train_root, Y_train_vowel, Y_train_consonant = make_labels(3)
train_nn(train_0, 3)

Y_train_root, Y_train_vowel, Y_train_consonant = make_labels(4)
train_nn(train_0, 4)

del train_0

print('train_1 is being processed')
train_1 = pd.read_parquet('/kaggle/input/bengaliai-cv19/train_image_data_1.parquet')

Y_train_root, Y_train_vowel, Y_train_consonant = make_labels(5)
train_nn(train_1, 0)

Y_train_root, Y_train_vowel, Y_train_consonant = make_labels(6)
train_nn(train_1, 1)

Y_train_root, Y_train_vowel, Y_train_consonant = make_labels(7)
train_nn(train_1, 2)

Y_train_root, Y_train_vowel, Y_train_consonant = make_labels(8)
train_nn(train_1, 3)

Y_train_root, Y_train_vowel, Y_train_consonant = make_labels(9)
train_nn(train_1, 4)

del train_1

print('train_2 is being processed')
train_2 = pd.read_parquet('/kaggle/input/bengaliai-cv19/train_image_data_2.parquet')

Y_train_root, Y_train_vowel, Y_train_consonant = make_labels(10)
train_nn(train_2, 0)

Y_train_root, Y_train_vowel, Y_train_consonant = make_labels(11)
train_nn(train_2, 1)

Y_train_root, Y_train_vowel, Y_train_consonant = make_labels(12)
train_nn(train_2, 2)

Y_train_root, Y_train_vowel, Y_train_consonant = make_labels(13)
train_nn(train_2, 3)

Y_train_root, Y_train_vowel, Y_train_consonant = make_labels(14)
train_nn(train_2, 4)

del train_2

print('train_3 is being processed')
train_3 = pd.read_parquet('/kaggle/input/bengaliai-cv19/train_image_data_3.parquet')

Y_train_root, Y_train_vowel, Y_train_consonant = make_labels(15)
train_nn(train_3, 0)

Y_train_root, Y_train_vowel, Y_train_consonant = make_labels(16)
train_nn(train_3, 1)

Y_train_root, Y_train_vowel, Y_train_consonant = make_labels(17)
train_nn(train_3, 2)

Y_train_root, Y_train_vowel, Y_train_consonant = make_labels(18)
train_nn(train_3, 3)

Y_train_root, Y_train_vowel, Y_train_consonant = make_labels(19)
train_nn(train_3, 4)

del train_3
torch.save(model.state_dict(), DEPLOYMENT_WEIGHTS)<jupyter_output><empty_output><jupyter_text>5. Evaluation<jupyter_code>def plot_loss(epoch_losses):
    plt.style.use('seaborn-whitegrid')
    plt.figure()
    
    plt.plot(np.arange(0, N_EPOCHS * 20), epoch_losses)
    
    plt.title('Train Loss')
    plt.xlabel('Epoch #')
    plt.ylabel('Loss')
    plt.show()    

plot_loss(epoch_losses)<jupyter_output><empty_output><jupyter_text>6. Deployment<jupyter_code>#keras
#prediction = model.predict(X_test)
data_loader_test = data.DataLoader(
    X_test,
    batch_size=BATCH_SIZE,
    num_workers=N_WORKERS,
    shuffle=False
)
model.load_state_dict(torch.load(DEPLOYMENT_WEIGHTS))
results_graph, results_vowel, results_conso = [], [], []
for epoch in range(N_EPOCHS):
    
    print('Epoch {}/{}'.format(epoch, N_EPOCHS - 1))
    print('-' * 10)
        
    model.eval()

    for step, batch in enumerate(tqdm(data_loader_test)):
        inputs = batch.to(device, dtype=torch.float)
        
        out_graph, out_vowel, out_conso = model(inputs)
        
        out_graph = F.softmax(out_graph, dim=1).data.cpu().numpy().argmax(axis=1)
        out_vowel = F.softmax(out_vowel, dim=1).data.cpu().numpy().argmax(axis=1)
        out_conso = F.softmax(out_conso, dim=1).data.cpu().numpy().argmax(axis=1)
            
            
        results_graph.append(out_graph)
        results_vowel.append(out_vowel)
        results_conso.append(out_conso)
            

results = []
results_graph = pd.DataFrame(results_graph)
results_vowel = pd.DataFrame(results_vowel)
results_conso = pd.DataFrame(results_conso)

for arg in range(sample_submission.shape[0] // 3 ):
    results.append(np.argmax(np.bincount(results_conso.iloc[:, arg])))
    results.append(np.argmax(np.bincount(results_graph.iloc[:, arg])))
    results.append(np.argmax(np.bincount(results_vowel.iloc[:, arg])))
results<jupyter_output><empty_output><jupyter_text>7. Submission<jupyter_code>#torch

submission = pd.concat([sample_submission.drop('target', axis=1), pd.Series(results)], names=['row_id', 'target'], axis=1)
submission.rename(columns={0: 'target'}, inplace=True)
submission
# keras

# for pred_index, value in enumerate(prediction):
#     for arg_index in range(3):
#         sample_submission['target'].iloc[pred_index+(3*arg_index)] = np.argmax(value, axis=1)[arg_index]
submission.to_csv('submission.csv', index=False)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup