


import math
import os
import unittest

from transformers import MegatronBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import (
        MODEL_FOR_PRETRAINING_MAPPING,
        MegatronBertForCausalLM,
        MegatronBertForMaskedLM,
        MegatronBertForMultipleChoice,
        MegatronBertForNextSentencePrediction,
        MegatronBertForPreTraining,
        MegatronBertForQuestionAnswering,
        MegatronBertForSequenceClassification,
        MegatronBertForTokenClassification,
        MegatronBertModel,
    )



class 							_UpperCamelCase :

   '''simple docstring'''







   def __init__(  self  : str       ,  _lowerCamelCase  : str       ,  _lowerCamelCase  : Dict=1_3       ,  _lowerCamelCase  : Dict=7       ,  _lowerCamelCase  : Any=True       ,  _lowerCamelCase  : Any=True       ,  _lowerCamelCase  : Optional[int]=True       ,  _lowerCamelCase  : Any=True       ,  _lowerCamelCase  : Dict=9_9       ,  _lowerCamelCase  : int=6_4       ,  _lowerCamelCase  : Dict=3_2       ,  _lowerCamelCase  : int=5       ,  _lowerCamelCase  : Union[str, Any]=4       ,  _lowerCamelCase  : Optional[int]=3_7       ,  _lowerCamelCase  : List[Any]="gelu"       ,  _lowerCamelCase  : Any=0.1       ,  _lowerCamelCase  : Dict=0.1       ,  _lowerCamelCase  : Union[str, Any]=5_1_2       ,  _lowerCamelCase  : List[str]=1_6       ,  _lowerCamelCase  : str=2       ,  _lowerCamelCase  : Optional[int]=0.02       ,  _lowerCamelCase  : Optional[int]=3       ,  _lowerCamelCase  : Optional[int]=4       ,  _lowerCamelCase  : Dict=None       ,  ):






         '''simple docstring'''



         __lowerCamelCase     :      Optional[Any]       						=						parent
         __lowerCamelCase     :      Optional[Any]       						=						batch_size
         __lowerCamelCase     :      str       						=						seq_length
         __lowerCamelCase     :      Union[str, Any]       						=						is_training
         __lowerCamelCase     :      Tuple       						=						use_input_mask
         __lowerCamelCase     :      int       						=						use_token_type_ids
         __lowerCamelCase     :      List[Any]       						=						use_labels
         __lowerCamelCase     :      List[Any]       						=						vocab_size
         __lowerCamelCase     :      Any       						=						hidden_size
         __lowerCamelCase     :      Tuple       						=						embedding_size
         __lowerCamelCase     :      List[Any]       						=						num_hidden_layers
         __lowerCamelCase     :      Optional[Any]       						=						num_attention_heads
         __lowerCamelCase     :      List[str]       						=						intermediate_size
         __lowerCamelCase     :      List[str]       						=						hidden_act
         __lowerCamelCase     :      Dict       						=						hidden_dropout_prob
         __lowerCamelCase     :      List[Any]       						=						attention_probs_dropout_prob
         __lowerCamelCase     :      Tuple       						=						max_position_embeddings
         __lowerCamelCase     :      Optional[int]       						=						type_vocab_size
         __lowerCamelCase     :      List[str]       						=						type_sequence_label_size
         __lowerCamelCase     :      Optional[int]       						=						initializer_range
         __lowerCamelCase     :      Any       						=						num_labels
         __lowerCamelCase     :      Any       						=						num_choices
         __lowerCamelCase     :      Optional[int]       						=						scope

   def   _snake_case    (  self  : Any							):






         '''simple docstring'''



         __lowerCamelCase     :      List[str]       						=						ids_tensor([self.batch_size, self.seq_length]       ,  self.vocab_size							)

         __lowerCamelCase     :      Union[str, Any]       						=						None
         if self.use_input_mask:
               __lowerCamelCase     :      Optional[Any]       						=						random_attention_mask([self.batch_size, self.seq_length]							)

         __lowerCamelCase     :      Dict       						=						None
         if self.use_token_type_ids:
               __lowerCamelCase     :      Optional[int]       						=						ids_tensor([self.batch_size, self.seq_length]       ,  self.type_vocab_size							)

         __lowerCamelCase     :      Dict       						=						None
         __lowerCamelCase     :      Tuple       						=						None
         __lowerCamelCase     :      Optional[Any]       						=						None
         if self.use_labels:
               __lowerCamelCase     :      str       						=						ids_tensor([self.batch_size]       ,  self.type_sequence_label_size							)
               __lowerCamelCase     :      str       						=						ids_tensor([self.batch_size, self.seq_length]       ,  self.num_labels							)
               __lowerCamelCase     :      int       						=						ids_tensor([self.batch_size]       ,  self.num_choices							)

         __lowerCamelCase     :      Optional[Any]       						=						self.get_config()

         return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

   def   _snake_case    (  self  : str							):






         '''simple docstring'''



         return MegatronBertConfig(
             vocab_size=self.vocab_size       ,  hidden_size=self.hidden_size       ,  num_hidden_layers=self.num_hidden_layers       ,  num_attention_heads=self.num_attention_heads       ,  intermediate_size=self.intermediate_size       ,  embedding_size=self.embedding_size       ,  hidden_act=self.hidden_act       ,  hidden_dropout_prob=self.hidden_dropout_prob       ,  attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,  max_position_embeddings=self.max_position_embeddings       ,  type_vocab_size=self.type_vocab_size       ,  is_decoder=__snake_case       ,  initializer_range=self.initializer_range       ,  )

   def   _snake_case    (  self  : int       ,  _lowerCamelCase  : List[str]       ,  _lowerCamelCase  : int       ,  _lowerCamelCase  : Tuple       ,  _lowerCamelCase  : int       ,  _lowerCamelCase  : Optional[int]       ,  _lowerCamelCase  : Union[str, Any]       ,  _lowerCamelCase  : Optional[int]							):






         '''simple docstring'''



         __lowerCamelCase     :      Dict       						=						MegatronBertModel(config=__snake_case							)
         model.to(__snake_case							)
         model.eval()
         __lowerCamelCase     :      Tuple       						=						model(__snake_case       ,  attention_mask=__snake_case       ,  token_type_ids=__snake_case							)
         __lowerCamelCase     :      str       						=						model(__snake_case       ,  token_type_ids=__snake_case							)
         __lowerCamelCase     :      Optional[int]       						=						model(__snake_case							)

         self.parent.assertEqual(result.last_hidden_state.shape       ,  (self.batch_size, self.seq_length, self.hidden_size)							)
         self.parent.assertEqual(result.pooler_output.shape       ,  (self.batch_size, self.hidden_size)							)

   def   _snake_case    (  self  : int       ,  _lowerCamelCase  : Optional[int]       ,  _lowerCamelCase  : Union[str, Any]       ,  _lowerCamelCase  : str       ,  _lowerCamelCase  : Any       ,  _lowerCamelCase  : int       ,  _lowerCamelCase  : Dict       ,  _lowerCamelCase  : Union[str, Any]							):






         '''simple docstring'''



         __lowerCamelCase     :      Dict       						=						MegatronBertForMaskedLM(config=__snake_case							)
         model.to(__snake_case							)
         model.eval()
         __lowerCamelCase     :      List[Any]       						=						model(__snake_case       ,  attention_mask=__snake_case       ,  token_type_ids=__snake_case       ,  labels=__snake_case							)
         self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.seq_length, self.vocab_size)							)

   def   _snake_case    (  self  : Tuple       ,  _lowerCamelCase  : int       ,  _lowerCamelCase  : str       ,  _lowerCamelCase  : Any       ,  _lowerCamelCase  : List[str]       ,  _lowerCamelCase  : str       ,  _lowerCamelCase  : Any       ,  _lowerCamelCase  : List[Any]							):






         '''simple docstring'''



         __lowerCamelCase     :      Any       						=						MegatronBertForCausalLM(config=__snake_case							)
         model.to(__snake_case							)
         model.eval()
         __lowerCamelCase     :      str       						=						model(__snake_case       ,  attention_mask=__snake_case       ,  token_type_ids=__snake_case       ,  labels=__snake_case							)
         self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.seq_length, self.vocab_size)							)

   def   _snake_case    (  self  : str       ,  _lowerCamelCase  : Optional[int]       ,  _lowerCamelCase  : Union[str, Any]       ,  _lowerCamelCase  : Optional[int]       ,  _lowerCamelCase  : str       ,  _lowerCamelCase  : Optional[int]       ,  _lowerCamelCase  : List[str]       ,  _lowerCamelCase  : str							):






         '''simple docstring'''



         __lowerCamelCase     :      Dict       						=						MegatronBertForNextSentencePrediction(config=__snake_case							)
         model.to(__snake_case							)
         model.eval()
         __lowerCamelCase     :      int       						=						model(
             __snake_case       ,  attention_mask=__snake_case       ,  token_type_ids=__snake_case       ,  labels=__snake_case       ,  )
         self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, 2)							)

   def   _snake_case    (  self  : List[str]       ,  _lowerCamelCase  : str       ,  _lowerCamelCase  : int       ,  _lowerCamelCase  : Tuple       ,  _lowerCamelCase  : Dict       ,  _lowerCamelCase  : List[str]       ,  _lowerCamelCase  : Optional[int]       ,  _lowerCamelCase  : Optional[int]							):






         '''simple docstring'''



         __lowerCamelCase     :      Dict       						=						MegatronBertForPreTraining(config=__snake_case							)
         model.to(__snake_case							)
         model.eval()
         __lowerCamelCase     :      Optional[Any]       						=						model(
             __snake_case       ,  attention_mask=__snake_case       ,  token_type_ids=__snake_case       ,  labels=__snake_case       ,  next_sentence_label=__snake_case       ,  )
         self.parent.assertEqual(result.prediction_logits.shape       ,  (self.batch_size, self.seq_length, self.vocab_size)							)
         self.parent.assertEqual(result.seq_relationship_logits.shape       ,  (self.batch_size, 2)							)

   def   _snake_case    (  self  : Optional[int]       ,  _lowerCamelCase  : List[str]       ,  _lowerCamelCase  : List[str]       ,  _lowerCamelCase  : Union[str, Any]       ,  _lowerCamelCase  : Union[str, Any]       ,  _lowerCamelCase  : Tuple       ,  _lowerCamelCase  : List[Any]       ,  _lowerCamelCase  : Optional[int]							):






         '''simple docstring'''



         __lowerCamelCase     :      Dict       						=						MegatronBertForQuestionAnswering(config=__snake_case							)
         model.to(__snake_case							)
         model.eval()
         __lowerCamelCase     :      Tuple       						=						model(
             __snake_case       ,  attention_mask=__snake_case       ,  token_type_ids=__snake_case       ,  start_positions=__snake_case       ,  end_positions=__snake_case       ,  )
         self.parent.assertEqual(result.start_logits.shape       ,  (self.batch_size, self.seq_length)							)
         self.parent.assertEqual(result.end_logits.shape       ,  (self.batch_size, self.seq_length)							)

   def   _snake_case    (  self  : Dict       ,  _lowerCamelCase  : Union[str, Any]       ,  _lowerCamelCase  : str       ,  _lowerCamelCase  : Any       ,  _lowerCamelCase  : int       ,  _lowerCamelCase  : int       ,  _lowerCamelCase  : Tuple       ,  _lowerCamelCase  : Tuple							):






         '''simple docstring'''



         __lowerCamelCase     :      int       						=						self.num_labels
         __lowerCamelCase     :      List[str]       						=						MegatronBertForSequenceClassification(__snake_case							)
         model.to(__snake_case							)
         model.eval()
         __lowerCamelCase     :      List[str]       						=						model(__snake_case       ,  attention_mask=__snake_case       ,  token_type_ids=__snake_case       ,  labels=__snake_case							)
         self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.num_labels)							)

   def   _snake_case    (  self  : List[str]       ,  _lowerCamelCase  : Optional[int]       ,  _lowerCamelCase  : str       ,  _lowerCamelCase  : int       ,  _lowerCamelCase  : Dict       ,  _lowerCamelCase  : Tuple       ,  _lowerCamelCase  : List[Any]       ,  _lowerCamelCase  : Dict							):






         '''simple docstring'''



         __lowerCamelCase     :      Optional[Any]       						=						self.num_labels
         __lowerCamelCase     :      int       						=						MegatronBertForTokenClassification(config=__snake_case							)
         model.to(__snake_case							)
         model.eval()
         __lowerCamelCase     :      Any       						=						model(__snake_case       ,  attention_mask=__snake_case       ,  token_type_ids=__snake_case       ,  labels=__snake_case							)
         self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.seq_length, self.num_labels)							)

   def   _snake_case    (  self  : Optional[Any]       ,  _lowerCamelCase  : Optional[Any]       ,  _lowerCamelCase  : List[str]       ,  _lowerCamelCase  : Dict       ,  _lowerCamelCase  : Union[str, Any]       ,  _lowerCamelCase  : Optional[Any]       ,  _lowerCamelCase  : Tuple       ,  _lowerCamelCase  : Union[str, Any]							):






         '''simple docstring'''



         __lowerCamelCase     :      Optional[Any]       						=						self.num_choices
         __lowerCamelCase     :      Tuple       						=						MegatronBertForMultipleChoice(config=__snake_case							)
         model.to(__snake_case							)
         model.eval()
         __lowerCamelCase     :      List[Any]       						=						input_ids.unsqueeze(1							).expand(-1       ,  self.num_choices       ,  -1							).contiguous()
         __lowerCamelCase     :      List[Any]       						=						token_type_ids.unsqueeze(1							).expand(-1       ,  self.num_choices       ,  -1							).contiguous()
         __lowerCamelCase     :      List[str]       						=						input_mask.unsqueeze(1							).expand(-1       ,  self.num_choices       ,  -1							).contiguous()
         __lowerCamelCase     :      Dict       						=						model(
             __snake_case       ,  attention_mask=__snake_case       ,  token_type_ids=__snake_case       ,  labels=__snake_case       ,  )
         self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.num_choices)							)


   def   _snake_case    (  self  : Dict							):






         '''simple docstring'''



         __lowerCamelCase     :      List[str]       						=						self.prepare_config_and_inputs()
         (
             __lowerCamelCase    
         )     :      Optional[Any]       						=						config_and_inputs
         __lowerCamelCase     :      List[Any]       						=						{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
         return config, inputs_dict



@require_torch
class 							_UpperCamelCase (						__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,unittest.TestCase						):

   '''simple docstring'''







   a_				:							List[Any]    = (
       (
           MegatronBertModel,
           MegatronBertForMaskedLM,
           MegatronBertForCausalLM,
           MegatronBertForMultipleChoice,
           MegatronBertForNextSentencePrediction,
           MegatronBertForPreTraining,
           MegatronBertForQuestionAnswering,
           MegatronBertForSequenceClassification,
           MegatronBertForTokenClassification,
       )
       if is_torch_available()
       else ()
   )
   a_				:							Optional[int]    = (
       {
           "feature-extraction": MegatronBertModel,
           "fill-mask": MegatronBertForMaskedLM,
           "question-answering": MegatronBertForQuestionAnswering,
           "text-classification": MegatronBertForSequenceClassification,
           "text-generation": MegatronBertForCausalLM,
           "token-classification": MegatronBertForTokenClassification,
           "zero-shot": MegatronBertForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )
   a_				:							int    = True
   # test_resize_embeddings = False
   a_				:							Optional[int]    = False

   def   _snake_case    (  self  : Any       ,  _lowerCamelCase  : Optional[Any]       ,  _lowerCamelCase  : str       ,  _lowerCamelCase  : List[Any]=False							):






         '''simple docstring'''



         __lowerCamelCase     :      Union[str, Any]       						=						super()._prepare_for_class(__snake_case       ,  __snake_case       ,  return_labels=__snake_case							)

         if return_labels:
               if model_class in get_values(__snake_case							):
                     __lowerCamelCase     :      int       						=						torch.zeros(
                         (self.model_tester.batch_size, self.model_tester.seq_length)       ,  dtype=torch.long       ,  device=__snake_case							)
                     __lowerCamelCase     :      str       						=						torch.zeros(
                         self.model_tester.batch_size       ,  dtype=torch.long       ,  device=__snake_case							)
         return inputs_dict

   def   _snake_case    (  self  : Optional[Any]							):






         '''simple docstring'''



         __lowerCamelCase     :      List[Any]       						=						MegatronBertModelTester(self							)
         __lowerCamelCase     :      Union[str, Any]       						=						ConfigTester(self       ,  config_class=__snake_case       ,  hidden_size=3_7							)

   def   _snake_case    (  self  : Dict							):






         '''simple docstring'''



         self.config_tester.run_common_tests()

   def   _snake_case    (  self  : Any							):






         '''simple docstring'''



         __lowerCamelCase     :      int       						=						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_megatron_bert_model(*__snake_case							)

   def   _snake_case    (  self  : List[str]							):






         '''simple docstring'''



         __lowerCamelCase     :      Optional[Any]       						=						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_megatron_bert_for_masked_lm(*__snake_case							)

   def   _snake_case    (  self  : Any							):






         '''simple docstring'''



         __lowerCamelCase     :      Dict       						=						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_megatron_bert_for_multiple_choice(*__snake_case							)

   def   _snake_case    (  self  : int							):






         '''simple docstring'''



         __lowerCamelCase     :      List[Any]       						=						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_megatron_bert_for_next_sequence_prediction(*__snake_case							)

   def   _snake_case    (  self  : List[Any]							):






         '''simple docstring'''



         __lowerCamelCase     :      Tuple       						=						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_megatron_bert_for_pretraining(*__snake_case							)

   def   _snake_case    (  self  : Union[str, Any]							):






         '''simple docstring'''



         __lowerCamelCase     :      Dict       						=						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_megatron_bert_for_question_answering(*__snake_case							)

   def   _snake_case    (  self  : int							):






         '''simple docstring'''



         __lowerCamelCase     :      Dict       						=						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_megatron_bert_for_sequence_classification(*__snake_case							)


   def   _snake_case    (  self  : Optional[int]							):






         '''simple docstring'''



         __lowerCamelCase     :      List[Any]       						=						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_megatron_bert_for_token_classification(*__snake_case							)





def   _UpperCAmelCase       (     UpperCAmelCase      :			Optional[Any]      ):
      """simple docstring"""




      return torch.tensor(
          a_						,	dtype=torch.long						,	device=a_						,	)


__UpperCamelCase    :						int     =							1E-4




@require_torch
@require_sentencepiece
@require_tokenizers
class 							_UpperCamelCase (						unittest.TestCase						):

   '''simple docstring'''







   @slow
   @unittest.skip("""Model is not available."""							)
   def   _snake_case    (  self  : Union[str, Any]							):






         '''simple docstring'''



         __lowerCamelCase     :      Union[str, Any]       						=						'''nvidia/megatron-bert-uncased-345m'''
         if "MYDIR" in os.environ:
               __lowerCamelCase     :      Any       						=						os.path.join(os.environ["""MYDIR"""]       ,  __snake_case							)
         __lowerCamelCase     :      Union[str, Any]       						=						MegatronBertModel.from_pretrained(__snake_case							)
         model.to(__snake_case							)
         model.half()
         __lowerCamelCase     :      List[str]       						=						_long_tensor([[1_0_1, 7_1_1_0, 1_0_0_5, 1_0_5_6, 2_0_2_3, 1_1_3_3_3, 1_7_4_1_3, 1_0_2_9, 1_0_2]]							)
         with torch.no_grad():
               __lowerCamelCase     :      Any       						=						model(__snake_case							)[0]
         __lowerCamelCase     :      Tuple       						=						torch.Size((1, 9, 1_0_2_4)							)
         self.assertEqual(output.shape       ,  __snake_case							)

         __lowerCamelCase     :      str       						=						[-0.6_040, -0.2_517, -0.1_025, 0.3_420, -0.6_758, -0.0_017, -0.1_089, -0.1_990, 0.5_728]
         for ii in range(3							):
               for jj in range(3							):
                     __lowerCamelCase     :      int       						=						output[0, ii, jj]
                     __lowerCamelCase     :      str       						=						expected[3 * ii + jj]
                     __lowerCamelCase     :      Union[str, Any]       						=						'''ii={} jj={} a={} b={}'''.format(__snake_case       ,  __snake_case       ,  __snake_case       ,  __snake_case							)
                     self.assertTrue(math.isclose(__snake_case       ,  __snake_case       ,  rel_tol=__snake_case       ,  abs_tol=__snake_case							)       ,  msg=__snake_case							)





import re
import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin



class 							_UpperCamelCase (						A						):

   '''simple docstring'''







   a_				:							List[Any]    = ["image_processor", "tokenizer"]
   a_				:							Tuple    = "AutoImageProcessor"
   a_				:							Tuple    = "AutoTokenizer"

   def __init__(  self  : Union[str, Any]       ,  _lowerCamelCase  : List[Any]=None       ,  _lowerCamelCase  : Any=None       ,  **_lowerCamelCase  : List[str]							):






         '''simple docstring'''



         __lowerCamelCase     :      Tuple       						=						None
         if "feature_extractor" in kwargs:
               warnings.warn(
                   """The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"""
                   """ instead."""       ,  _lowerCamelCase       ,  )
               __lowerCamelCase     :      List[Any]       						=						kwargs.pop("""feature_extractor"""							)

         __lowerCamelCase     :      Optional[Any]       						=						image_processor if image_processor is not None else feature_extractor
         if image_processor is None:
               raise ValueError("""You need to specify an `image_processor`."""							)
         if tokenizer is None:
               raise ValueError("""You need to specify a `tokenizer`."""							)

         super().__init__(_lowerCamelCase       ,  _lowerCamelCase							)
         __lowerCamelCase     :      List[Any]       						=						self.image_processor
         __lowerCamelCase     :      Any       						=						False

   def __call__(  self  : Union[str, Any]       ,  *_lowerCamelCase  : str       ,  **_lowerCamelCase  : Dict							):






         '''simple docstring'''



         if self._in_target_context_manager:
               return self.current_processor(*_lowerCamelCase       ,  **_lowerCamelCase							)

         __lowerCamelCase     :      List[str]       						=						kwargs.pop("""images"""       ,  _lowerCamelCase							)
         __lowerCamelCase     :      List[str]       						=						kwargs.pop("""text"""       ,  _lowerCamelCase							)
         if len(_lowerCamelCase							) > 0:
               __lowerCamelCase     :      Tuple       						=						args[0]
               __lowerCamelCase     :      List[Any]       						=						args[1:]

         if images is None and text is None:
               raise ValueError("""You need to specify either an `images` or `text` input to process."""							)

         if images is not None:
               __lowerCamelCase     :      Tuple       						=						self.image_processor(_lowerCamelCase       ,  *_lowerCamelCase       ,  **_lowerCamelCase							)
         if text is not None:
               __lowerCamelCase     :      int       						=						self.tokenizer(_lowerCamelCase       ,  **_lowerCamelCase							)

         if text is None:
               return inputs
         elif images is None:
               return encodings
         else:
               __lowerCamelCase     :      Optional[int]       						=						encodings["""input_ids"""]
               return inputs

   def   _snake_case    (  self  : List[Any]       ,  *_lowerCamelCase  : Tuple       ,  **_lowerCamelCase  : Union[str, Any]							):






         '''simple docstring'''



         return self.tokenizer.batch_decode(*_lowerCamelCase       ,  **_lowerCamelCase							)

   def   _snake_case    (  self  : List[str]       ,  *_lowerCamelCase  : Optional[int]       ,  **_lowerCamelCase  : int							):






         '''simple docstring'''



         return self.tokenizer.decode(*_lowerCamelCase       ,  **_lowerCamelCase							)

   @contextmanager
   def   _snake_case    (  self  : Optional[Any]							):






         '''simple docstring'''



         warnings.warn(
             """`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your """
             """labels by using the argument `text` of the regular `__call__` method (either in the same call as """
             """your images inputs, or in a separate call."""							)
         __lowerCamelCase     :      Any       						=						True
         __lowerCamelCase     :      int       						=						self.tokenizer
         yield
         __lowerCamelCase     :      Dict       						=						self.image_processor
         __lowerCamelCase     :      List[Any]       						=						False

   def   _snake_case    (  self  : List[str]       ,  _lowerCamelCase  : str       ,  _lowerCamelCase  : List[str]=False       ,  _lowerCamelCase  : List[Any]=None							):






         '''simple docstring'''



         if added_vocab is None:
               __lowerCamelCase     :      List[str]       						=						self.tokenizer.get_added_vocab()

         __lowerCamelCase     :      List[Any]       						=						{}

         while tokens:
               __lowerCamelCase     :      Dict       						=						re.search(R"""<s_(.*?)>"""       ,  _lowerCamelCase       ,  re.IGNORECASE							)
               if start_token is None:
                     break
               __lowerCamelCase     :      Dict       						=						start_token.group(1							)
               __lowerCamelCase     :      List[str]       						=						re.search(RF"""</s_{key}>"""       ,  _lowerCamelCase       ,  re.IGNORECASE							)
               __lowerCamelCase     :      Optional[int]       						=						start_token.group()
               if end_token is None:
                     __lowerCamelCase     :      int       						=						tokens.replace(_lowerCamelCase       ,  """"""							)
               else:
                     __lowerCamelCase     :      Dict       						=						end_token.group()
                     __lowerCamelCase     :      Optional[Any]       						=						re.escape(_lowerCamelCase							)
                     __lowerCamelCase     :      List[str]       						=						re.escape(_lowerCamelCase							)
                     __lowerCamelCase     :      Any       						=						re.search(F"""{start_token_escaped}(.*?){end_token_escaped}"""       ,  _lowerCamelCase       ,  re.IGNORECASE							)
                     if content is not None:
                           __lowerCamelCase     :      Union[str, Any]       						=						content.group(1							).strip()
                           if r"<s_" in content and r"</s_" in content:  # non-leaf node
                                 __lowerCamelCase     :      Union[str, Any]       						=						self.tokenajson(_lowerCamelCase       ,  is_inner_value=_lowerCamelCase       ,  added_vocab=_lowerCamelCase							)
                                 if value:
                                       if len(_lowerCamelCase							) == 1:
                                             __lowerCamelCase     :      Optional[int]       						=						value[0]
                                       __lowerCamelCase     :      Union[str, Any]       						=						value
                           else:  # leaf nodes
                                 __lowerCamelCase     :      List[Any]       						=						[]
                                 for leaf in content.split(R"""<sep/>"""							):
                                       __lowerCamelCase     :      Optional[Any]       						=						leaf.strip()
                                       if leaf in added_vocab and leaf[0] == "<" and leaf[-2:] == "/>":
                                             __lowerCamelCase     :      str       						=						leaf[1:-2]  # for categorical special tokens
                                       output[key].append(_lowerCamelCase							)
                                 if len(output[key]							) == 1:
                                       __lowerCamelCase     :      Tuple       						=						output[key][0]

                     __lowerCamelCase     :      Tuple       						=						tokens[tokens.find(_lowerCamelCase							) + len(_lowerCamelCase							) :].strip()
                     if tokens[:6] == r"<sep/>":  # non-leaf nodes
                           return [output] + self.tokenajson(tokens[6:]       ,  is_inner_value=_lowerCamelCase       ,  added_vocab=_lowerCamelCase							)

         if len(_lowerCamelCase							):
               return [output] if is_inner_value else output
         else:
               return [] if is_inner_value else {"text_sequence": tokens}

   @property
   def   _snake_case    (  self  : Tuple							):






         '''simple docstring'''



         warnings.warn(
             """`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead."""       ,  _lowerCamelCase       ,  )
         return self.image_processor_class


   @property
   def   _snake_case    (  self  : List[Any]							):






         '''simple docstring'''



         warnings.warn(
             """`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead."""       ,  _lowerCamelCase       ,  )
         return self.image_processor



