




import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
	from transformers.pipelines.conversational import Conversation


__lowerCamelCase   :						Optional[int]				=			logging.get_logger(__name__)

__lowerCamelCase   :						Optional[Any]				=			{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__lowerCamelCase   :						Dict				=			{
    '''vocab_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/vocab.json''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/vocab.json''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/vocab.json''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/vocab.json''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/merges.txt''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/merges.txt''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/merges.txt''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/merges.txt''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/tokenizer.json''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/tokenizer.json''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/tokenizer.json''',
    },
}

__lowerCamelCase   :						List[Any]				=			{
    '''gpt2''': 10_24,
    '''gpt2-medium''': 10_24,
    '''gpt2-large''': 10_24,
    '''gpt2-xl''': 10_24,
    '''distilgpt2''': 10_24,
}


class    lowerCamelCase    (		_lowerCamelCase						):






			'''simple docstring'''





			UpperCamelCase__								=VOCAB_FILES_NAMES
			UpperCamelCase__								=PRETRAINED_VOCAB_FILES_MAP
			UpperCamelCase__								=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			UpperCamelCase__								=['''input_ids''', '''attention_mask''']
			UpperCamelCase__								=GPTaTokenizer

			def __init__( self			:   List[str]       ,						lowerCamelCase_			:   Tuple=None       ,						lowerCamelCase_			:   Tuple=None       ,						lowerCamelCase_			:   int=None       ,						lowerCamelCase_			:   Any="<|endoftext|>"       ,						lowerCamelCase_			:   Optional[int]="<|endoftext|>"       ,						lowerCamelCase_			:   int="<|endoftext|>"       ,						lowerCamelCase_			:   Optional[Any]=False       ,						**lowerCamelCase_			:   Union[str, Any]       ,						)						->   Any:
								super().__init__(
								    lowerCamelCase_       ,						lowerCamelCase_       ,						tokenizer_file=lowerCamelCase_       ,						unk_token=lowerCamelCase_       ,						bos_token=lowerCamelCase_       ,						eos_token=lowerCamelCase_       ,						add_prefix_space=lowerCamelCase_       ,						**lowerCamelCase_       ,						)

								__magic_name__							:	str    							=							kwargs.pop('''add_bos_token'''       ,						lowerCamelCase_  )

								__magic_name__							:	int    							=							json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()  )
								if pre_tok_state.get('''add_prefix_space'''       ,						lowerCamelCase_  ) != add_prefix_space:
													__magic_name__							:	List[Any]    							=							getattr(lowerCamelCase_       ,						pre_tok_state.pop('''type'''  )  )
													__magic_name__							:	str    							=							add_prefix_space
													__magic_name__							:	Dict    							=							pre_tok_class(**lowerCamelCase_  )

								__magic_name__							:	List[str]    							=							add_prefix_space

			def 							UpperCAmelCase__       ( self			:   List[Any]       ,						*lowerCamelCase_			:   Optional[Any]       ,						**lowerCamelCase_			:   Any  )						->   BatchEncoding:
								__magic_name__							:	Optional[int]    							=							kwargs.get('''is_split_into_words'''       ,						lowerCamelCase_  )
								assert self.add_prefix_space or not is_split_into_words, (
								    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
								    "to use it with pretokenized inputs."
								)

								return super()._batch_encode_plus(*lowerCamelCase_       ,						**lowerCamelCase_  )

			def 							UpperCAmelCase__       ( self			:   Any       ,						*lowerCamelCase_			:   List[str]       ,						**lowerCamelCase_			:   List[str]  )						->   BatchEncoding:
								__magic_name__							:	List[Any]    							=							kwargs.get('''is_split_into_words'''       ,						lowerCamelCase_  )

								assert self.add_prefix_space or not is_split_into_words, (
								    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
								    "to use it with pretokenized inputs."
								)

								return super()._encode_plus(*lowerCamelCase_       ,						**lowerCamelCase_  )

			def 							UpperCAmelCase__       ( self			:   Any       ,						lowerCamelCase_			:   str       ,						lowerCamelCase_			:   Optional[str] = None  )						->   Tuple[str]:
								__magic_name__							:	List[str]    							=							self._tokenizer.model.save(lowerCamelCase_       ,						name=lowerCamelCase_  )
								return tuple(lowerCamelCase_  )


			def 							UpperCAmelCase__       ( self			:   Tuple       ,						lowerCamelCase_			:   "Conversation"  )						->   List[int]:
								__magic_name__							:	List[Any]    							=							[]
								for is_user, text in conversation.iter_texts():
													input_ids.extend(self.encode(lowerCamelCase_       ,						add_special_tokens=lowerCamelCase_  ) + [self.eos_token_id]  )

								if len(lowerCamelCase_  ) > self.model_max_length:
													__magic_name__							:	List[Any]    							=							input_ids[-self.model_max_length :]
								return input_ids









# flake8: noqa
# Lint as: python3

from typing import Dict, List, Optional, Type

from .. import config
from ..utils import logging
from .formatting import (
    ArrowFormatter,
    CustomFormatter,
    Formatter,
    PandasFormatter,
    PythonFormatter,
    TensorFormatter,
    format_table,
    query_table,
)
from .np_formatter import NumpyFormatter


__lowerCamelCase   :						int				=			logging.get_logger(__name__)

__lowerCamelCase   :						Dict[Optional[str], Type[Formatter]]				=			{}
__lowerCamelCase   :						Dict[Optional[str], str]				=			{}
__lowerCamelCase   :						Dict[Optional[str], Exception]				=			{}







def 		lowercase__   (		__A:	type							,__A:	Optional[str]							,__A:	Optional[List[str]] = None							,):





					'''simple docstring'''






					__magic_name__							:	Union[str, Any]    							=							aliases if aliases is not None else []
					if format_type in _FORMAT_TYPES:
										logger.warning(
										    F'''Overwriting format type \'{format_type}\' ({_FORMAT_TYPES[format_type].__name__} -> {formatter_cls.__name__})'''				)
					__magic_name__							:	Tuple    							=							formatter_cls
					for alias in set(aliases + [format_type]				):
										if alias in _FORMAT_TYPES_ALIASES:
															logger.warning(
															    F'''Overwriting format type alias \'{alias}\' ({_FORMAT_TYPES_ALIASES[alias]} -> {format_type})'''				)
										__magic_name__							:	Optional[int]    							=							format_type







def 		lowercase__   (		__A:	Exception							,__A:	Optional[str]							,__A:	Optional[List[str]] = None				):





					'''simple docstring'''






					__magic_name__							:	Union[str, Any]    							=							aliases if aliases is not None else []
					for alias in set(aliases + [format_type]				):
										__magic_name__							:	List[str]    							=							unavailable_error


# Here we define all the available formatting functions that can be used by `Dataset.set_format`
_register_formatter(PythonFormatter, None, aliases=['''python'''])
_register_formatter(ArrowFormatter, '''arrow''', aliases=['''pa''', '''pyarrow'''])
_register_formatter(NumpyFormatter, '''numpy''', aliases=['''np'''])
_register_formatter(PandasFormatter, '''pandas''', aliases=['''pd'''])
_register_formatter(CustomFormatter, '''custom''')

if config.TORCH_AVAILABLE:
	from .torch_formatter import TorchFormatter

	_register_formatter(TorchFormatter, '''torch''', aliases=['''pt''', '''pytorch'''])
else:
	__lowerCamelCase   :						str				=			ValueError('''PyTorch needs to be installed to be able to return PyTorch tensors.''')
	_register_unavailable_formatter(_torch_error, '''torch''', aliases=['''pt''', '''pytorch'''])

if config.TF_AVAILABLE:
	from .tf_formatter import TFFormatter

	_register_formatter(TFFormatter, '''tensorflow''', aliases=['''tf'''])
else:
	__lowerCamelCase   :						int				=			ValueError('''Tensorflow needs to be installed to be able to return Tensorflow tensors.''')
	_register_unavailable_formatter(_tf_error, '''tensorflow''', aliases=['''tf'''])

if config.JAX_AVAILABLE:
	from .jax_formatter import JaxFormatter

	_register_formatter(JaxFormatter, '''jax''', aliases=[])
else:
	__lowerCamelCase   :						str				=			ValueError('''JAX needs to be installed to be able to return JAX arrays.''')
	_register_unavailable_formatter(_jax_error, '''jax''', aliases=[])







def 		lowercase__   (		__A:	Optional[str]				):





					'''simple docstring'''






					if format_type in _FORMAT_TYPES_ALIASES:
										return _FORMAT_TYPES_ALIASES[format_type]
					else:
										return format_type







def 		lowercase__   (		__A:	Optional[str]							,**__A:	Any				):





					'''simple docstring'''






					__magic_name__							:	Tuple    							=							get_format_type_from_alias(__A				)
					if format_type in _FORMAT_TYPES:
										return _FORMAT_TYPES[format_type](**__A				)
					if format_type in _FORMAT_TYPES_ALIASES_UNAVAILABLE:
										raise _FORMAT_TYPES_ALIASES_UNAVAILABLE[format_type]
					else:
										raise ValueError(
										    F'''Return type should be None or selected in {list(type for type in _FORMAT_TYPES.keys() if type != None				)}, but got \'{format_type}\''''				)



