





"""simple docstring"""




import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
		from transformers.pipelines.conversational import Conversation


a		:				Tuple  			=    logging.get_logger(__name__)

a		:				Optional[Any]  			=    {'''tokenizer_file''': '''tokenizer.json'''}

a		:				str  			=    {
    '''tokenizer_file''': {
        '''bigscience/tokenizer''': '''https://huggingface.co/bigscience/tokenizer/blob/main/tokenizer.json''',
        '''bigscience/bloom-560m''': '''https://huggingface.co/bigscience/bloom-560m/blob/main/tokenizer.json''',
        '''bigscience/bloom-1b1''': '''https://huggingface.co/bigscience/bloom-1b1/blob/main/tokenizer.json''',
        '''bigscience/bloom-1b7''': '''https://huggingface.co/bigscience/bloom-1b7/blob/main/tokenizer.json''',
        '''bigscience/bloom-3b''': '''https://huggingface.co/bigscience/bloom-3b/blob/main/tokenizer.json''',
        '''bigscience/bloom-7b1''': '''https://huggingface.co/bigscience/bloom-7b1/blob/main/tokenizer.json''',
        '''bigscience/bloom''': '''https://huggingface.co/bigscience/bloom/blob/main/tokenizer.json''',
    },
}


class  __UpperCamelCase       (		a__		):
							lowerCamelCase     :      Tuple         =VOCAB_FILES_NAMES
							lowerCamelCase     :      str         =PRETRAINED_VOCAB_FILES_MAP
							lowerCamelCase     :      List[Any]         =["""input_ids""", """attention_mask"""]
							lowerCamelCase     :      Dict         =None






							def __init__(     self   ,      lowerCAmelCase__=None   ,      lowerCAmelCase__=None   ,      lowerCAmelCase__=None   ,      lowerCAmelCase__="<unk>"   ,      lowerCAmelCase__="<s>"   ,      lowerCAmelCase__="</s>"   ,      lowerCAmelCase__="<pad>"   ,      lowerCAmelCase__=False   ,      lowerCAmelCase__=False   ,      **lowerCAmelCase__   ,      )			->				Optional[int]:
								super().__init__(
								    lowerCAmelCase__   ,      lowerCAmelCase__   ,      tokenizer_file=lowerCAmelCase__   ,      unk_token=lowerCAmelCase__   ,      bos_token=lowerCAmelCase__   ,      eos_token=lowerCAmelCase__   ,      pad_token=lowerCAmelCase__   ,      add_prefix_space=lowerCAmelCase__   ,      clean_up_tokenization_spaces=lowerCAmelCase__   ,      **lowerCAmelCase__   ,      )
								a   :	Any			=       json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()   )
								if pre_tok_state.get("add_prefix_space"   ,      lowerCAmelCase__   ) != add_prefix_space:
									a   :	Union[str, Any]			=       getattr(lowerCAmelCase__   ,      pre_tok_state.pop("type"   )   )
									a   :	Optional[int]			=       add_prefix_space
									a   :	Any			=       pre_tok_class(**lowerCAmelCase__   )

								a   :	Optional[int]			=       add_prefix_space






							def 					__a     (     self   ,      *lowerCAmelCase__   ,      **lowerCAmelCase__   )			->				BatchEncoding:
								a   :	Dict			=       kwargs.get("is_split_into_words"   ,      lowerCAmelCase__   )
								if not (self.add_prefix_space or not is_split_into_words):
									raise Exception(
									    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with"""
									    " pretokenized inputs."   )

								return super()._batch_encode_plus(*lowerCAmelCase__   ,      **lowerCAmelCase__   )






							def 					__a     (     self   ,      *lowerCAmelCase__   ,      **lowerCAmelCase__   )			->				BatchEncoding:
								a   :	List[Any]			=       kwargs.get("is_split_into_words"   ,      lowerCAmelCase__   )

								if not (self.add_prefix_space or not is_split_into_words):
									raise Exception(
									    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with"""
									    " pretokenized inputs."   )

								return super()._encode_plus(*lowerCAmelCase__   ,      **lowerCAmelCase__   )






							def 					__a     (     self   ,      lowerCAmelCase__   ,      lowerCAmelCase__ = None   )			->				Tuple[str]:
								a   :	Union[str, Any]			=       self._tokenizer.model.save(lowerCAmelCase__   ,      name=lowerCAmelCase__   )
								return tuple(lowerCAmelCase__   )






							def 					__a     (     self   ,      lowerCAmelCase__   )			->				List[int]:
								a   :	Dict			=       []
								for is_user, text in conversation.iter_texts():
									input_ids.extend(self.encode(lowerCAmelCase__   ,      add_special_tokens=lowerCAmelCase__   ) + [self.eos_token_id]   )

								if len(lowerCAmelCase__   ) > self.model_max_length:
									a   :	int			=       input_ids[-self.model_max_length :]
								return input_ids









"""simple docstring"""




from ..utils import DummyObject, requires_backends


class  __UpperCamelCase       (		metaclass=a__		):
							lowerCamelCase     :      Optional[Any]         =["""transformers""", """torch""", """note_seq"""]






							def __init__(     self   ,      *lowerCAmelCase__   ,      **lowerCAmelCase__   )			->				Union[str, Any]:
								requires_backends(self   ,      ["transformers", "torch", "note_seq"]   )






							@classmethod
							def 					__a     (     cls   ,      *lowerCAmelCase__   ,      **lowerCAmelCase__   )			->				Any:
								requires_backends(cls   ,      ["transformers", "torch", "note_seq"]   )






							@classmethod
							def 					__a     (     cls   ,      *lowerCAmelCase__   ,      **lowerCAmelCase__   )			->				int:
								requires_backends(cls   ,      ["transformers", "torch", "note_seq"]   )


