feat: implemented positional interpolation

modeling_bert.py

@@ -787,7 +787,8 @@ class JinaBertEncoder(nn.Module):
             # Device catch-up
             self.alibi = self.alibi.to(hidden_states.device)
 
-        alibi_bias = self.alibi[:, :, :seqlen, :seqlen]
+        unpadded_seqlens = torch.sum(attention_mask, dim=1).unsqueeze(1).unsqueeze(1).unsqueeze(1)
+        alibi_bias = self.alibi[:, :, :seqlen, :seqlen] * 512 / unpadded_seqlens
         if self.gradient_checkpointing and self.training:
             if use_cache:
                 logger.warning_once(