

import gc
import unittest

from transformers import MODEL_FOR_MASKED_LM_MAPPING, TF_MODEL_FOR_MASKED_LM_MAPPING, FillMaskPipeline, pipeline
from transformers.pipelines import PipelineException
from transformers.testing_utils import (
    is_pipeline_test,
    is_torch_available,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_gpu,
    slow,
)

from .test_pipelines_common import ANY






@is_pipeline_test
class 						__magic_name__    (       unittest.TestCase    ):
 UpperCAmelCase							    =						MODEL_FOR_MASKED_LM_MAPPING
 UpperCAmelCase							    =						TF_MODEL_FOR_MASKED_LM_MAPPING


 def _UpperCamelCase	(						self		: int    )    ->      Optional[Any]:
   super().tearDown()
   # clean-up as much as possible GPU memory occupied by PyTorch
   gc.collect()
   if is_torch_available():
     import torch

     torch.cuda.empty_cache()


 @require_tf
 def _UpperCamelCase	(						self		: Any    )    ->      Optional[Any]:
   UpperCAmelCase					      =  pipeline(task="fill-mask"			,						model="sshleifer/tiny-distilroberta-base"			,						top_k=2			,						framework="tf"    )
   UpperCAmelCase					      =  unmasker("My name is <mask>"    )
   self.assertEqual(
       nested_simplify(lowercase__			,						decimals=6    )			,						[
           {"sequence": "My name is grouped", "score": 2.1e-0_5, "token": 3_8_0_1_5, "token_str": " grouped"},
           {"sequence": "My name is accuser", "score": 2.1e-0_5, "token": 2_5_5_0_6, "token_str": " accuser"},
       ]			,						)

   UpperCAmelCase					      =  unmasker("The largest city in France is <mask>"    )
   self.assertEqual(
       nested_simplify(lowercase__			,						decimals=6    )			,						[
           {
               "sequence": "The largest city in France is grouped",
               "score": 2.1e-0_5,
               "token": 3_8_0_1_5,
               "token_str": " grouped",
           },
           {
               "sequence": "The largest city in France is accuser",
               "score": 2.1e-0_5,
               "token": 2_5_5_0_6,
               "token_str": " accuser",
           },
       ]			,						)

   UpperCAmelCase					      =  unmasker("My name is <mask>"			,						targets=[" Patrick", " Clara", " Teven"]			,						top_k=3    )
   self.assertEqual(
       nested_simplify(lowercase__			,						decimals=6    )			,						[
           {"sequence": "My name is Clara", "score": 2e-0_5, "token": 1_3_6_0_6, "token_str": " Clara"},
           {"sequence": "My name is Patrick", "score": 2e-0_5, "token": 3_4_9_9, "token_str": " Patrick"},
           {"sequence": "My name is Te", "score": 1.9e-0_5, "token": 2_9_4_1, "token_str": " Te"},
       ]			,						)


 @require_torch
 def _UpperCamelCase	(						self		: Tuple    )    ->      Union[str, Any]:
   UpperCAmelCase					      =  pipeline(task="fill-mask"			,						model="sshleifer/tiny-distilroberta-base"			,						top_k=2			,						framework="pt"    )

   UpperCAmelCase					      =  unmasker("My name is <mask>"    )
   self.assertEqual(
       nested_simplify(lowercase__			,						decimals=6    )			,						[
           {"sequence": "My name is Maul", "score": 2.2e-0_5, "token": 3_5_6_7_6, "token_str": " Maul"},
           {"sequence": "My name isELS", "score": 2.2e-0_5, "token": 1_6_4_1_6, "token_str": "ELS"},
       ]			,						)

   UpperCAmelCase					      =  unmasker("The largest city in France is <mask>"    )
   self.assertEqual(
       nested_simplify(lowercase__			,						decimals=6    )			,						[
           {
               "sequence": "The largest city in France is Maul",
               "score": 2.2e-0_5,
               "token": 3_5_6_7_6,
               "token_str": " Maul",
           },
           {"sequence": "The largest city in France isELS", "score": 2.2e-0_5, "token": 1_6_4_1_6, "token_str": "ELS"},
       ]			,						)

   UpperCAmelCase					      =  unmasker("My name is <mask>"			,						targets=[" Patrick", " Clara", " Teven"]			,						top_k=3    )
   self.assertEqual(
       nested_simplify(lowercase__			,						decimals=6    )			,						[
           {"sequence": "My name is Patrick", "score": 2.1e-0_5, "token": 3_4_9_9, "token_str": " Patrick"},
           {"sequence": "My name is Te", "score": 2e-0_5, "token": 2_9_4_1, "token_str": " Te"},
           {"sequence": "My name is Clara", "score": 2e-0_5, "token": 1_3_6_0_6, "token_str": " Clara"},
       ]			,						)

   UpperCAmelCase					      =  unmasker("My name is <mask> <mask>"			,						top_k=2    )

   self.assertEqual(
       nested_simplify(lowercase__			,						decimals=6    )			,						[
           [
               {
                   "score": 2.2e-0_5,
                   "token": 3_5_6_7_6,
                   "token_str": " Maul",
                   "sequence": "<s>My name is Maul<mask></s>",
               },
               {"score": 2.2e-0_5, "token": 1_6_4_1_6, "token_str": "ELS", "sequence": "<s>My name isELS<mask></s>"},
           ],
           [
               {
                   "score": 2.2e-0_5,
                   "token": 3_5_6_7_6,
                   "token_str": " Maul",
                   "sequence": "<s>My name is<mask> Maul</s>",
               },
               {"score": 2.2e-0_5, "token": 1_6_4_1_6, "token_str": "ELS", "sequence": "<s>My name is<mask>ELS</s>"},
           ],
       ]			,						)


 @require_torch_gpu
 def _UpperCamelCase	(						self		: Union[str, Any]    )    ->      Dict:
   UpperCAmelCase					      =  pipeline("fill-mask"			,						model="hf-internal-testing/tiny-random-distilbert"			,						device=0			,						framework="pt"    )

   # convert model to fp16
   pipe.model.half()

   UpperCAmelCase					      =  pipe("Paris is the [MASK] of France."    )
   # We actually don't care about the result, we just want to make sure
   # it works, meaning the float16 tensor got casted back to float32
   # for postprocessing.
   self.assertIsInstance(lowercase__			,						lowercase__    )


 @slow
 @require_torch
 def _UpperCamelCase	(						self		: Optional[Any]    )    ->      Optional[int]:
   UpperCAmelCase					      =  pipeline(task="fill-mask"			,						model="distilroberta-base"			,						top_k=2			,						framework="pt"    )
   self.run_large_test(lowercase__    )


 @slow
 @require_tf
 def _UpperCamelCase	(						self		: List[str]    )    ->      int:
   UpperCAmelCase					      =  pipeline(task="fill-mask"			,						model="distilroberta-base"			,						top_k=2			,						framework="tf"    )
   self.run_large_test(lowercase__    )


 def _UpperCamelCase	(						self		: List[Any]			,						lowerCAmelCase__		: Optional[Any]    )    ->      str:
   UpperCAmelCase					      =  unmasker("My name is <mask>"    )
   self.assertEqual(
       nested_simplify(lowercase__    )			,						[
           {"sequence": "My name is John", "score": 0.008, "token": 6_1_0, "token_str": " John"},
           {"sequence": "My name is Chris", "score": 0.007, "token": 1_5_7_3, "token_str": " Chris"},
       ]			,						)
   UpperCAmelCase					      =  unmasker("The largest city in France is <mask>"    )
   self.assertEqual(
       nested_simplify(lowercase__    )			,						[
           {
               "sequence": "The largest city in France is Paris",
               "score": 0.251,
               "token": 2_2_0_1,
               "token_str": " Paris",
           },
           {
               "sequence": "The largest city in France is Lyon",
               "score": 0.214,
               "token": 1_2_7_9_0,
               "token_str": " Lyon",
           },
       ]			,						)

   UpperCAmelCase					      =  unmasker("My name is <mask>"			,						targets=[" Patrick", " Clara", " Teven"]			,						top_k=3    )
   self.assertEqual(
       nested_simplify(lowercase__    )			,						[
           {"sequence": "My name is Patrick", "score": 0.005, "token": 3_4_9_9, "token_str": " Patrick"},
           {"sequence": "My name is Clara", "score": 0.000, "token": 1_3_6_0_6, "token_str": " Clara"},
           {"sequence": "My name is Te", "score": 0.000, "token": 2_9_4_1, "token_str": " Te"},
       ]			,						)


 @require_torch
 def _UpperCamelCase	(						self		: Optional[int]    )    ->      Optional[Any]:
   UpperCAmelCase					      =  pipeline(task="fill-mask"			,						model="sshleifer/tiny-distilroberta-base"			,						framework="pt"    )
   UpperCAmelCase					      =  None
   UpperCAmelCase					      =  None
   self.run_pipeline_test(lowercase__			,						[]    )


 @require_tf
 def _UpperCamelCase	(						self		: List[str]    )    ->      Dict:
   UpperCAmelCase					      =  pipeline(task="fill-mask"			,						model="sshleifer/tiny-distilroberta-base"			,						framework="tf"    )
   UpperCAmelCase					      =  None
   UpperCAmelCase					      =  None
   self.run_pipeline_test(lowercase__			,						[]    )


 def _UpperCamelCase	(						self		: Optional[int]			,						lowerCAmelCase__		: str			,						lowerCAmelCase__		: List[Any]			,						lowerCAmelCase__		: List[Any]    )    ->      str:
   if tokenizer is None or tokenizer.mask_token_id is None:
     self.skipTest("The provided tokenizer has no mask token, (probably reformer or wav2vec2)"    )

   UpperCAmelCase					      =  FillMaskPipeline(model=lowercase__			,						tokenizer=lowercase__    )
   UpperCAmelCase					      =  [
       f"This is another {tokenizer.mask_token} test",
   ]
   return fill_masker, examples


 def _UpperCamelCase	(						self		: Dict			,						lowerCAmelCase__		: Optional[Any]			,						lowerCAmelCase__		: List[Any]    )    ->      Union[str, Any]:
   UpperCAmelCase					      =  fill_masker.tokenizer
   UpperCAmelCase					      =  fill_masker.model

   UpperCAmelCase					      =  fill_masker(
       f"This is a {tokenizer.mask_token}"			,						)
   self.assertEqual(
       lowercase__			,						[
           {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
           {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
           {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
           {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
           {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
       ]			,						)

   UpperCAmelCase					      =  fill_masker([f"This is a {tokenizer.mask_token}"]    )
   self.assertEqual(
       lowercase__			,						[
           {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
           {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
           {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
           {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
           {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
       ]			,						)

   UpperCAmelCase					      =  fill_masker([f"This is a {tokenizer.mask_token}", f"Another {tokenizer.mask_token} great test."]    )
   self.assertEqual(
       lowercase__			,						[
           [
               {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
               {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
               {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
               {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
               {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
           ],
           [
               {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
               {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
               {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
               {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
               {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
           ],
       ]			,						)

   with self.assertRaises(lowercase__    ):
     fill_masker([None]    )
   # No mask_token is not supported
   with self.assertRaises(lowercase__    ):
     fill_masker("This is"    )

   self.run_test_top_k(lowercase__			,						lowercase__    )
   self.run_test_targets(lowercase__			,						lowercase__    )
   self.run_test_top_k_targets(lowercase__			,						lowercase__    )
   self.fill_mask_with_duplicate_targets_and_top_k(lowercase__			,						lowercase__    )
   self.fill_mask_with_multiple_masks(lowercase__			,						lowercase__    )


 def _UpperCamelCase	(						self		: Any			,						lowerCAmelCase__		: Optional[int]			,						lowerCAmelCase__		: Dict    )    ->      List[Any]:
   UpperCAmelCase					      =  tokenizer.get_vocab()
   UpperCAmelCase					      =  sorted(vocab.keys()    )[:2]
   # Pipeline argument
   UpperCAmelCase					      =  FillMaskPipeline(model=lowercase__			,						tokenizer=lowercase__			,						targets=lowercase__    )
   UpperCAmelCase					      =  fill_masker(f"This is a {tokenizer.mask_token}"    )
   self.assertEqual(
       lowercase__			,						[
           {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
           {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
       ]			,						)
   UpperCAmelCase					      =  {vocab[el] for el in targets}
   self.assertEqual({el["token"] for el in outputs}			,						lowercase__    )
   UpperCAmelCase					      =  [tokenizer.decode([x]    ) for x in target_ids]
   self.assertEqual({el["token_str"] for el in outputs}			,						set(lowercase__    )    )

   # Call argument
   UpperCAmelCase					      =  FillMaskPipeline(model=lowercase__			,						tokenizer=lowercase__    )
   UpperCAmelCase					      =  fill_masker(f"This is a {tokenizer.mask_token}"			,						targets=lowercase__    )
   self.assertEqual(
       lowercase__			,						[
           {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
           {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
       ]			,						)
   UpperCAmelCase					      =  {vocab[el] for el in targets}
   self.assertEqual({el["token"] for el in outputs}			,						lowercase__    )
   UpperCAmelCase					      =  [tokenizer.decode([x]    ) for x in target_ids]
   self.assertEqual({el["token_str"] for el in outputs}			,						set(lowercase__    )    )

   # Score equivalence
   UpperCAmelCase					      =  fill_masker(f"This is a {tokenizer.mask_token}"			,						targets=lowercase__    )
   UpperCAmelCase					      =  [top_mask["""token_str"""] for top_mask in outputs]
   UpperCAmelCase					      =  [top_mask["""score"""] for top_mask in outputs]

   # For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
   if set(lowercase__    ) == set(lowercase__    ):
     UpperCAmelCase					      =  fill_masker(f"This is a {tokenizer.mask_token}"			,						targets=lowercase__    )
     UpperCAmelCase					      =  [top_mask["""score"""] for top_mask in unmasked_targets]
     self.assertEqual(nested_simplify(lowercase__    )			,						nested_simplify(lowercase__    )    )

   # Raises with invalid
   with self.assertRaises(lowercase__    ):
     UpperCAmelCase					      =  fill_masker(f"This is a {tokenizer.mask_token}"			,						targets=[]    )
   # For some tokenizers, `""` is actually in the vocabulary and the expected error won't raised
   if "" not in tokenizer.get_vocab():
     with self.assertRaises(lowercase__    ):
       UpperCAmelCase					      =  fill_masker(f"This is a {tokenizer.mask_token}"			,						targets=[""]    )
     with self.assertRaises(lowercase__    ):
       UpperCAmelCase					      =  fill_masker(f"This is a {tokenizer.mask_token}"			,						targets=""    )


 def _UpperCamelCase	(						self		: Optional[Any]			,						lowerCAmelCase__		: Union[str, Any]			,						lowerCAmelCase__		: str    )    ->      Optional[int]:
   UpperCAmelCase					      =  FillMaskPipeline(model=lowercase__			,						tokenizer=lowercase__			,						top_k=2    )
   UpperCAmelCase					      =  fill_masker(f"This is a {tokenizer.mask_token}"    )
   self.assertEqual(
       lowercase__			,						[
           {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
           {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
       ]			,						)

   UpperCAmelCase					      =  FillMaskPipeline(model=lowercase__			,						tokenizer=lowercase__    )
   UpperCAmelCase					      =  fill_masker(f"This is a {tokenizer.mask_token}"			,						top_k=2    )
   self.assertEqual(
       lowercase__			,						[
           {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
           {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
       ]			,						)
   self.assertEqual(nested_simplify(lowercase__    )			,						nested_simplify(lowercase__    )    )


 def _UpperCamelCase	(						self		: int			,						lowerCAmelCase__		: Union[str, Any]			,						lowerCAmelCase__		: int    )    ->      List[Any]:
   UpperCAmelCase					      =  tokenizer.get_vocab()
   UpperCAmelCase					      =  FillMaskPipeline(model=lowercase__			,						tokenizer=lowercase__    )

   # top_k=2, ntargets=3
   UpperCAmelCase					      =  sorted(vocab.keys()    )[:3]
   UpperCAmelCase					      =  fill_masker(f"This is a {tokenizer.mask_token}"			,						top_k=2			,						targets=lowercase__    )

   # If we use the most probably targets, and filter differently, we should still
   # have the same results
   UpperCAmelCase					      =  [el["""token_str"""] for el in sorted(lowercase__			,						key=lambda lowerCAmelCase__    : x["score"]			,						reverse=lowercase__    )]
   # For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
   if set(lowercase__    ).issubset(lowercase__    ):
     UpperCAmelCase					      =  fill_masker(f"This is a {tokenizer.mask_token}"			,						top_k=3			,						targets=lowercase__    )
     # They should yield exactly the same result
     self.assertEqual(nested_simplify(lowercase__    )			,						nested_simplify(lowercase__    )    )


 def _UpperCamelCase	(						self		: Optional[Any]			,						lowerCAmelCase__		: List[Any]			,						lowerCAmelCase__		: List[str]    )    ->      Any:
   UpperCAmelCase					      =  FillMaskPipeline(model=lowercase__			,						tokenizer=lowercase__    )
   UpperCAmelCase					      =  tokenizer.get_vocab()
   # String duplicates + id duplicates
   UpperCAmelCase					      =  sorted(vocab.keys()    )[:3]
   UpperCAmelCase					      =  [targets[0], targets[1], targets[0], targets[2], targets[1]]
   UpperCAmelCase					      =  fill_masker(f"My name is {tokenizer.mask_token}"			,						targets=lowercase__			,						top_k=1_0    )

   # The target list contains duplicates, so we can't output more
   # than them
   self.assertEqual(len(lowercase__    )			,						3    )


 def _UpperCamelCase	(						self		: List[str]			,						lowerCAmelCase__		: int			,						lowerCAmelCase__		: Dict    )    ->      Union[str, Any]:
   UpperCAmelCase					      =  FillMaskPipeline(model=lowercase__			,						tokenizer=lowercase__    )

   UpperCAmelCase					      =  fill_masker(
       f"This is a {tokenizer.mask_token} {tokenizer.mask_token} {tokenizer.mask_token}"			,						top_k=2    )
   self.assertEqual(
       lowercase__			,						[
           [
               {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
               {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
           ],
           [
               {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
               {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
           ],
           [
               {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
               {"sequence": ANY(lowercase__    ), "score": ANY(lowercase__    ), "token": ANY(lowercase__    ), "token_str": ANY(lowercase__    )},
           ],
       ]			,						)






from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase__       = logging.get_logger(__name__)

lowerCAmelCase__       = {
    "unc-nlp/lxmert-base-uncased": "https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/config.json",
}






class 						__magic_name__    (       _snake_case    ):
 UpperCAmelCase							    =						"""lxmert"""
 UpperCAmelCase							    =						{}


 def __init__(						self		: int			,						lowerCAmelCase__		: Any=3_0_5_2_2			,						lowerCAmelCase__		: List[str]=7_6_8			,						lowerCAmelCase__		: Union[str, Any]=1_2			,						lowerCAmelCase__		: List[Any]=9_5_0_0			,						lowerCAmelCase__		: Any=1_6_0_0			,						lowerCAmelCase__		: Union[str, Any]=4_0_0			,						lowerCAmelCase__		: Tuple=3_0_7_2			,						lowerCAmelCase__		: Dict="gelu"			,						lowerCAmelCase__		: Tuple=0.1			,						lowerCAmelCase__		: Tuple=0.1			,						lowerCAmelCase__		: int=5_1_2			,						lowerCAmelCase__		: List[str]=2			,						lowerCAmelCase__		: List[str]=0.02			,						lowerCAmelCase__		: str=1e-1_2			,						lowerCAmelCase__		: str=9			,						lowerCAmelCase__		: int=5			,						lowerCAmelCase__		: Optional[int]=5			,						lowerCAmelCase__		: List[Any]=2_0_4_8			,						lowerCAmelCase__		: Any=4			,						lowerCAmelCase__		: Dict=6.67			,						lowerCAmelCase__		: Any=True			,						lowerCAmelCase__		: Union[str, Any]=True			,						lowerCAmelCase__		: Any=True			,						lowerCAmelCase__		: Tuple=True			,						lowerCAmelCase__		: Optional[Any]=True			,						lowerCAmelCase__		: Optional[int]=True			,						lowerCAmelCase__		: Tuple=True			,						**lowerCAmelCase__		: List[Any]			,						)    ->      Dict:
   UpperCAmelCase					      =  vocab_size
   UpperCAmelCase					      =  hidden_size
   UpperCAmelCase					      =  num_attention_heads
   UpperCAmelCase					      =  hidden_act
   UpperCAmelCase					      =  intermediate_size
   UpperCAmelCase					      =  hidden_dropout_prob
   UpperCAmelCase					      =  attention_probs_dropout_prob
   UpperCAmelCase					      =  max_position_embeddings
   UpperCAmelCase					      =  type_vocab_size
   UpperCAmelCase					      =  initializer_range
   UpperCAmelCase					      =  layer_norm_eps
   UpperCAmelCase					      =  num_qa_labels
   UpperCAmelCase					      =  num_object_labels
   UpperCAmelCase					      =  num_attr_labels
   UpperCAmelCase					      =  l_layers
   UpperCAmelCase					      =  x_layers
   UpperCAmelCase					      =  r_layers
   UpperCAmelCase					      =  visual_feat_dim
   UpperCAmelCase					      =  visual_pos_dim
   UpperCAmelCase					      =  visual_loss_normalizer
   UpperCAmelCase					      =  task_matched
   UpperCAmelCase					      =  task_mask_lm
   UpperCAmelCase					      =  task_obj_predict
   UpperCAmelCase					      =  task_qa
   UpperCAmelCase					      =  visual_obj_loss
   UpperCAmelCase					      =  visual_attr_loss
   UpperCAmelCase					      =  visual_feat_loss
   UpperCAmelCase					      =  {"vision": r_layers, "cross_encoder": x_layers, "language": l_layers}
   super().__init__(**lowerCAmelCase__    )



