

from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_herbert import HerbertTokenizer


__UpperCAmelCase    				=     logging.get_logger(__name__)

__UpperCAmelCase    				=     {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

__UpperCAmelCase    				=     {
    "vocab_file": {
        "allegro/herbert-base-cased": "https://huggingface.co/allegro/herbert-base-cased/resolve/main/vocab.json"
    },
    "merges_file": {
        "allegro/herbert-base-cased": "https://huggingface.co/allegro/herbert-base-cased/resolve/main/merges.txt"
    },
}

__UpperCAmelCase    				=     {"allegro/herbert-base-cased": 5_1_4}
__UpperCAmelCase    				=     {}





class A__    ( _A ):



 """simple docstring"""







 _lowercase     :					int      =						VOCAB_FILES_NAMES
 _lowercase     :					Optional[Any]      =						PRETRAINED_VOCAB_FILES_MAP
 _lowercase     :					Dict      =						PRETRAINED_INIT_CONFIGURATION
 _lowercase     :					List[Any]      =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _lowercase     :					Optional[Any]      =						HerbertTokenizer

 def __init__(		self :	Any     ,						A_ :	str=None     ,						A_ :	Optional[int]=None     ,						A_ :	Union[str, Any]=None     ,						A_ :	str="<s>"     ,						A_ :	int="<unk>"     ,						A_ :	List[Any]="<pad>"     ,						A_ :	str="<mask>"     ,						A_ :	Any="</s>"     ,						**A_ :	str     ,						):






      '''simple docstring'''




      super().__init__(
          A_     ,						A_     ,						tokenizer_file=A_     ,						cls_token=A_     ,						unk_token=A_     ,						pad_token=A_     ,						mask_token=A_     ,						sep_token=A_     ,						**A_     ,						)

 def    __magic_name__       (		self :	int     ,						A_ :	List[Any]     ,						A_ :	Optional[Any] = None       ):






      '''simple docstring'''




      _lowerCAmelCase   :					Optional[int]        =    [self.cls_token_id]
      _lowerCAmelCase   :					Any        =    [self.sep_token_id]
      if token_ids_a is None:
           return cls + token_ids_a + sep

      return cls + token_ids_a + sep + token_ids_a + sep

 def    __magic_name__       (		self :	str     ,						A_ :	Tuple     ,						A_ :	str = None     ,						A_ :	int = False       ):






      '''simple docstring'''




      if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=A_     ,						token_ids_a=A_     ,						already_has_special_tokens=A_       )

      if token_ids_a is None:
           return [1] + ([0] * len(A_       )) + [1]
      return [1] + ([0] * len(A_       )) + [1] + ([0] * len(A_       )) + [1]

 def    __magic_name__       (		self :	List[Any]     ,						A_ :	Tuple     ,						A_ :	Dict = None       ):






      '''simple docstring'''




      _lowerCAmelCase   :					Any        =    [self.sep_token_id]
      _lowerCAmelCase   :					Union[str, Any]        =    [self.cls_token_id]

      if token_ids_a is None:
           return len(cls + token_ids_a + sep       ) * [0]
      return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]


 def    __magic_name__       (		self :	Union[str, Any]     ,						A_ :	Optional[Any]     ,						A_ :	str = None       ):






      '''simple docstring'''




      _lowerCAmelCase   :					List[str]        =    self._tokenizer.model.save(A_     ,						name=A_       )
      return tuple(A_       )









from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__UpperCAmelCase    				=     {
    'configuration_lxmert': ['LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'LxmertConfig'],
    'tokenization_lxmert': ['LxmertTokenizer'],
}

try:
  if not is_tokenizers_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __UpperCAmelCase    				=     ['LxmertTokenizerFast']

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __UpperCAmelCase    				=     [
      'LxmertEncoder',
      'LxmertForPreTraining',
      'LxmertForQuestionAnswering',
      'LxmertModel',
      'LxmertPreTrainedModel',
      'LxmertVisualFeatureEncoder',
      'LxmertXLayer',
  ]

try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __UpperCAmelCase    				=     [
      'TF_LXMERT_PRETRAINED_MODEL_ARCHIVE_LIST',
      'TFLxmertForPreTraining',
      'TFLxmertMainLayer',
      'TFLxmertModel',
      'TFLxmertPreTrainedModel',
      'TFLxmertVisualFeatureEncoder',
  ]


if TYPE_CHECKING:
  from .configuration_lxmert import LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP, LxmertConfig
  from .tokenization_lxmert import LxmertTokenizer

  try:
    if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_lxmert_fast import LxmertTokenizerFast

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_lxmert import (
        LxmertEncoder,
        LxmertForPreTraining,
        LxmertForQuestionAnswering,
        LxmertModel,
        LxmertPreTrainedModel,
        LxmertVisualFeatureEncoder,
        LxmertXLayer,
    )

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_tf_lxmert import (
        TF_LXMERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFLxmertForPreTraining,
        TFLxmertMainLayer,
        TFLxmertModel,
        TFLxmertPreTrainedModel,
        TFLxmertVisualFeatureEncoder,
    )

else:
  import sys

  __UpperCAmelCase    				=     _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)






