'''simple docstring'''






import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
    from transformers.pipelines.conversational import Conversation


SCREAMING_SNAKE_CASE_	   =			logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_	   =			{'tokenizer_file': 'tokenizer.json'}

SCREAMING_SNAKE_CASE_	   =			{
    'tokenizer_file': {
        'bigscience/tokenizer': 'https://huggingface.co/bigscience/tokenizer/blob/main/tokenizer.json',
        'bigscience/bloom-560m': 'https://huggingface.co/bigscience/bloom-560m/blob/main/tokenizer.json',
        'bigscience/bloom-1b1': 'https://huggingface.co/bigscience/bloom-1b1/blob/main/tokenizer.json',
        'bigscience/bloom-1b7': 'https://huggingface.co/bigscience/bloom-1b7/blob/main/tokenizer.json',
        'bigscience/bloom-3b': 'https://huggingface.co/bigscience/bloom-3b/blob/main/tokenizer.json',
        'bigscience/bloom-7b1': 'https://huggingface.co/bigscience/bloom-7b1/blob/main/tokenizer.json',
        'bigscience/bloom': 'https://huggingface.co/bigscience/bloom/blob/main/tokenizer.json',
    },
}




class    a (       __lowerCAmelCase				):



  """simple docstring"""
  __lowerCAmelCase				      =					VOCAB_FILES_NAMES
  __lowerCAmelCase				      =					PRETRAINED_VOCAB_FILES_MAP
  __lowerCAmelCase				      =					["""input_ids""", """attention_mask"""]
  __lowerCAmelCase				      =					None







  def __init__(							self   ,    snake_case_=None   ,    snake_case_=None   ,    snake_case_=None   ,    snake_case_="<unk>"   ,    snake_case_="<s>"   ,    snake_case_="</s>"   ,    snake_case_="<pad>"   ,    snake_case_=False   ,    snake_case_=False   ,    **snake_case_   ,    ):

    '''simple docstring'''







    super().__init__(
        snake_case_   ,    snake_case_   ,    tokenizer_file=snake_case_   ,    unk_token=snake_case_   ,    bos_token=snake_case_   ,    eos_token=snake_case_   ,    pad_token=snake_case_   ,    add_prefix_space=snake_case_   ,    clean_up_tokenization_spaces=snake_case_   ,    **snake_case_   ,    )
    __UpperCAmelCase:					List[Any]		= json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()					)
    if pre_tok_state.get("""add_prefix_space"""   ,    snake_case_					) != add_prefix_space:
      __UpperCAmelCase:					Tuple		= getattr(snake_case_   ,    pre_tok_state.pop("""type"""					)					)
      __UpperCAmelCase:					int		= add_prefix_space
      __UpperCAmelCase:					str		= pre_tok_class(**snake_case_					)

    __UpperCAmelCase:					Optional[Any]		= add_prefix_space







  def    lowercase_  (							self   ,    *snake_case_   ,    **snake_case_					):

    '''simple docstring'''







    __UpperCAmelCase:					Dict		= kwargs.get("""is_split_into_words"""   ,    snake_case_					)
    if not (self.add_prefix_space or not is_split_into_words):
      raise Exception(
          F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with'''
          """ pretokenized inputs."""					)

    return super()._batch_encode_plus(*snake_case_   ,    **snake_case_					)







  def    lowercase_  (							self   ,    *snake_case_   ,    **snake_case_					):

    '''simple docstring'''







    __UpperCAmelCase:					str		= kwargs.get("""is_split_into_words"""   ,    snake_case_					)

    if not (self.add_prefix_space or not is_split_into_words):
      raise Exception(
          F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with'''
          """ pretokenized inputs."""					)

    return super()._encode_plus(*snake_case_   ,    **snake_case_					)







  def    lowercase_  (							self   ,    snake_case_   ,    snake_case_ = None					):

    '''simple docstring'''







    __UpperCAmelCase:					Any		= self._tokenizer.model.save(snake_case_   ,    name=snake_case_					)
    return tuple(snake_case_					)







  def    lowercase_  (							self   ,    snake_case_					):

    '''simple docstring'''







    __UpperCAmelCase:					Optional[int]		= []
    for is_user, text in conversation.iter_texts():
      input_ids.extend(self.encode(snake_case_   ,    add_special_tokens=snake_case_					) + [self.eos_token_id]					)

    if len(snake_case_					) > self.model_max_length:
      __UpperCAmelCase:					Dict		= input_ids[-self.model_max_length :]
    return input_ids
'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


SCREAMING_SNAKE_CASE_	   =			{
    'configuration_electra': ['ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP', 'ElectraConfig', 'ElectraOnnxConfig'],
    'tokenization_electra': ['ElectraTokenizer'],
}

try:
    if not is_tokenizers_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    SCREAMING_SNAKE_CASE_	   =			['ElectraTokenizerFast']

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    SCREAMING_SNAKE_CASE_	   =			[
        'ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST',
        'ElectraForCausalLM',
        'ElectraForMaskedLM',
        'ElectraForMultipleChoice',
        'ElectraForPreTraining',
        'ElectraForQuestionAnswering',
        'ElectraForSequenceClassification',
        'ElectraForTokenClassification',
        'ElectraModel',
        'ElectraPreTrainedModel',
        'load_tf_weights_in_electra',
    ]

try:
    if not is_tf_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    SCREAMING_SNAKE_CASE_	   =			[
        'TF_ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST',
        'TFElectraForMaskedLM',
        'TFElectraForMultipleChoice',
        'TFElectraForPreTraining',
        'TFElectraForQuestionAnswering',
        'TFElectraForSequenceClassification',
        'TFElectraForTokenClassification',
        'TFElectraModel',
        'TFElectraPreTrainedModel',
    ]

try:
    if not is_flax_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    SCREAMING_SNAKE_CASE_	   =			[
        'FlaxElectraForCausalLM',
        'FlaxElectraForMaskedLM',
        'FlaxElectraForMultipleChoice',
        'FlaxElectraForPreTraining',
        'FlaxElectraForQuestionAnswering',
        'FlaxElectraForSequenceClassification',
        'FlaxElectraForTokenClassification',
        'FlaxElectraModel',
        'FlaxElectraPreTrainedModel',
    ]


if TYPE_CHECKING:
    from .configuration_electra import ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP, ElectraConfig, ElectraOnnxConfig
    from .tokenization_electra import ElectraTokenizer

    try:
        if not is_tokenizers_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .tokenization_electra_fast import ElectraTokenizerFast

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_electra import (
            ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST,
            ElectraForCausalLM,
            ElectraForMaskedLM,
            ElectraForMultipleChoice,
            ElectraForPreTraining,
            ElectraForQuestionAnswering,
            ElectraForSequenceClassification,
            ElectraForTokenClassification,
            ElectraModel,
            ElectraPreTrainedModel,
            load_tf_weights_in_electra,
        )

    try:
        if not is_tf_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_tf_electra import (
            TF_ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST,
            TFElectraForMaskedLM,
            TFElectraForMultipleChoice,
            TFElectraForPreTraining,
            TFElectraForQuestionAnswering,
            TFElectraForSequenceClassification,
            TFElectraForTokenClassification,
            TFElectraModel,
            TFElectraPreTrainedModel,
        )

    try:
        if not is_flax_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_flax_electra import (
            FlaxElectraForCausalLM,
            FlaxElectraForMaskedLM,
            FlaxElectraForMultipleChoice,
            FlaxElectraForPreTraining,
            FlaxElectraForQuestionAnswering,
            FlaxElectraForSequenceClassification,
            FlaxElectraForTokenClassification,
            FlaxElectraModel,
            FlaxElectraPreTrainedModel,
        )

else:
    import sys

    SCREAMING_SNAKE_CASE_	   =			_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)