



'''simple docstring'''



import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
							from transformers.pipelines.conversational import Conversation


lowerCAmelCase	:		List[Any]				=   logging.get_logger(__name__)

lowerCAmelCase	:		Optional[Any]				=   {'''tokenizer_file''': '''tokenizer.json'''}

lowerCAmelCase	:		Optional[int]				=   {
    '''tokenizer_file''': {
        '''bigscience/tokenizer''': '''https://huggingface.co/bigscience/tokenizer/blob/main/tokenizer.json''',
        '''bigscience/bloom-560m''': '''https://huggingface.co/bigscience/bloom-560m/blob/main/tokenizer.json''',
        '''bigscience/bloom-1b1''': '''https://huggingface.co/bigscience/bloom-1b1/blob/main/tokenizer.json''',
        '''bigscience/bloom-1b7''': '''https://huggingface.co/bigscience/bloom-1b7/blob/main/tokenizer.json''',
        '''bigscience/bloom-3b''': '''https://huggingface.co/bigscience/bloom-3b/blob/main/tokenizer.json''',
        '''bigscience/bloom-7b1''': '''https://huggingface.co/bigscience/bloom-7b1/blob/main/tokenizer.json''',
        '''bigscience/bloom''': '''https://huggingface.co/bigscience/bloom/blob/main/tokenizer.json''',
    },
}






class 					UpperCAmelCase__	(		UpperCamelCase__ ):
					a      :     Dict        =							VOCAB_FILES_NAMES
					a      :     Optional[Any]        =							PRETRAINED_VOCAB_FILES_MAP
					a      :     List[Any]        =							["""input_ids""", """attention_mask"""]
					a      :     Tuple        =							None



					def __init__(    self			,				UpperCamelCase=None			,				UpperCamelCase=None			,				UpperCamelCase=None			,				UpperCamelCase="<unk>"			,				UpperCamelCase="<s>"			,				UpperCamelCase="</s>"			,				UpperCamelCase="<pad>"			,				UpperCamelCase=False			,				UpperCamelCase=False			,				**UpperCamelCase			,				)     ->			Any:
						super().__init__(
						    UpperCamelCase			,				UpperCamelCase			,				tokenizer_file=UpperCamelCase			,				unk_token=UpperCamelCase			,				bos_token=UpperCamelCase			,				eos_token=UpperCamelCase			,				pad_token=UpperCamelCase			,				add_prefix_space=UpperCamelCase			,				clean_up_tokenization_spaces=UpperCamelCase			,				**UpperCamelCase			,				)
						__lowerCAmelCase												=  json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()     )
						if pre_tok_state.get("add_prefix_space"			,				UpperCamelCase     ) != add_prefix_space:
							__lowerCAmelCase												=  getattr(UpperCamelCase			,				pre_tok_state.pop("type"     )     )
							__lowerCAmelCase												=  add_prefix_space
							__lowerCAmelCase												=  pre_tok_class(**UpperCamelCase     )

						__lowerCAmelCase												=  add_prefix_space



					def        UpperCAmelCase_ (    self			,				*UpperCamelCase			,				**UpperCamelCase     )     ->			BatchEncoding:
						__lowerCAmelCase												=  kwargs.get("is_split_into_words"			,				UpperCamelCase     )
						if not (self.add_prefix_space or not is_split_into_words):
							raise Exception(
							    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with'''
							    " pretokenized inputs."     )

						return super()._batch_encode_plus(*UpperCamelCase			,				**UpperCamelCase     )



					def        UpperCAmelCase_ (    self			,				*UpperCamelCase			,				**UpperCamelCase     )     ->			BatchEncoding:
						__lowerCAmelCase												=  kwargs.get("is_split_into_words"			,				UpperCamelCase     )

						if not (self.add_prefix_space or not is_split_into_words):
							raise Exception(
							    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with'''
							    " pretokenized inputs."     )

						return super()._encode_plus(*UpperCamelCase			,				**UpperCamelCase     )



					def        UpperCAmelCase_ (    self			,				UpperCamelCase			,				UpperCamelCase = None     )     ->			Tuple[str]:
						__lowerCAmelCase												=  self._tokenizer.model.save(UpperCamelCase			,				name=UpperCamelCase     )
						return tuple(UpperCamelCase     )



					def        UpperCAmelCase_ (    self			,				UpperCamelCase     )     ->			List[int]:
						__lowerCAmelCase												=  []
						for is_user, text in conversation.iter_texts():
							input_ids.extend(self.encode(UpperCamelCase			,				add_special_tokens=UpperCamelCase     ) + [self.eos_token_id]     )

						if len(UpperCamelCase     ) > self.model_max_length:
							__lowerCAmelCase												=  input_ids[-self.model_max_length :]
						return input_ids




'''simple docstring'''



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCAmelCase	:		Union[str, Any]				=   {
    '''configuration_distilbert''': [
        '''DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''DistilBertConfig''',
        '''DistilBertOnnxConfig''',
    ],
    '''tokenization_distilbert''': ['''DistilBertTokenizer'''],
}

try:
	if not is_tokenizers_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	lowerCAmelCase	:		Tuple				=   ['''DistilBertTokenizerFast''']

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	lowerCAmelCase	:		Tuple				=   [
	    '''DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
	    '''DistilBertForMaskedLM''',
	    '''DistilBertForMultipleChoice''',
	    '''DistilBertForQuestionAnswering''',
	    '''DistilBertForSequenceClassification''',
	    '''DistilBertForTokenClassification''',
	    '''DistilBertModel''',
	    '''DistilBertPreTrainedModel''',
	]

try:
	if not is_tf_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	lowerCAmelCase	:		List[str]				=   [
	    '''TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
	    '''TFDistilBertForMaskedLM''',
	    '''TFDistilBertForMultipleChoice''',
	    '''TFDistilBertForQuestionAnswering''',
	    '''TFDistilBertForSequenceClassification''',
	    '''TFDistilBertForTokenClassification''',
	    '''TFDistilBertMainLayer''',
	    '''TFDistilBertModel''',
	    '''TFDistilBertPreTrainedModel''',
	]

try:
	if not is_flax_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	lowerCAmelCase	:		Optional[int]				=   [
	    '''FlaxDistilBertForMaskedLM''',
	    '''FlaxDistilBertForMultipleChoice''',
	    '''FlaxDistilBertForQuestionAnswering''',
	    '''FlaxDistilBertForSequenceClassification''',
	    '''FlaxDistilBertForTokenClassification''',
	    '''FlaxDistilBertModel''',
	    '''FlaxDistilBertPreTrainedModel''',
	]


if TYPE_CHECKING:
	from .configuration_distilbert import (
	    DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
	    DistilBertConfig,
	    DistilBertOnnxConfig,
	)
	from .tokenization_distilbert import DistilBertTokenizer

	try:
		if not is_tokenizers_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .tokenization_distilbert_fast import DistilBertTokenizerFast

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_distilbert import (
		    DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
		    DistilBertForMaskedLM,
		    DistilBertForMultipleChoice,
		    DistilBertForQuestionAnswering,
		    DistilBertForSequenceClassification,
		    DistilBertForTokenClassification,
		    DistilBertModel,
		    DistilBertPreTrainedModel,
		)

	try:
		if not is_tf_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_tf_distilbert import (
		    TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TFDistilBertForMaskedLM,
		    TFDistilBertForMultipleChoice,
		    TFDistilBertForQuestionAnswering,
		    TFDistilBertForSequenceClassification,
		    TFDistilBertForTokenClassification,
		    TFDistilBertMainLayer,
		    TFDistilBertModel,
		    TFDistilBertPreTrainedModel,
		)

	try:
		if not is_flax_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_flax_distilbert import (
		    FlaxDistilBertForMaskedLM,
		    FlaxDistilBertForMultipleChoice,
		    FlaxDistilBertForQuestionAnswering,
		    FlaxDistilBertForSequenceClassification,
		    FlaxDistilBertForTokenClassification,
		    FlaxDistilBertModel,
		    FlaxDistilBertPreTrainedModel,
		)

else:
	import sys

	lowerCAmelCase	:		List[str]				=   _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)