



import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


__lowerCamelCase     :							Any  							=				logging.getLogger(__name__)





@dataclass(frozen=A__						)
class  __magic_name__   :
		lowercase      :       str
		lowercase      :       str
		lowercase      :       Optional[str]          =None
		lowercase      :       Optional[str]          =None
		lowercase      :       Optional[str]          =None





@dataclass(frozen=A__						)
class  __magic_name__   :
		lowercase      :       List[int]
		lowercase      :       Optional[List[int]]          =None
		lowercase      :       Optional[List[int]]          =None
		lowercase      :       Optional[Union[int, float]]          =None
		lowercase      :       Optional[int]          =None


if is_torch_available():
		import torch
		from torch.utils.data import Dataset





		class  __magic_name__   (							A__						):
				lowercase      :       List[InputFeatures]






				def __init__(						self      :     int				,		UpperCamelCase__      :     str				,		UpperCamelCase__      :     PreTrainedTokenizer				,		UpperCamelCase__      :     str				,		UpperCamelCase__      :     Optional[int] = None				,		UpperCamelCase__      :     List[Any]=False				,		UpperCamelCase__      :     bool = False				,		)				->					Dict:
										'''simple docstring'''







										UpperCAmelCase         =     hans_processors[task]()

										UpperCAmelCase         =     os.path.join(
										    UpperCamelCase__				,		"cached_{}_{}_{}_{}".format(
										        "dev" if evaluate else "train"				,		tokenizer.__class__.__name__				,		str(UpperCamelCase__      )				,		UpperCamelCase__				,		)				,		)
										UpperCAmelCase         =     processor.get_labels()
										if tokenizer.__class__ in (
										    RobertaTokenizer,
										    RobertaTokenizerFast,
										    XLMRobertaTokenizer,
										    BartTokenizer,
										    BartTokenizerFast,
										):
																# HACK(label indices are swapped in RoBERTa pretrained model)
																UpperCAmelCase       ,		UpperCAmelCase         =     label_list[2], label_list[1]
										UpperCAmelCase         =     label_list

										# Make sure only the first process in distributed training processes the dataset,
										# and the others will use the cache.
										UpperCAmelCase         =     cached_features_file + ".lock"
										with FileLock(UpperCamelCase__      ):
																if os.path.exists(UpperCamelCase__      ) and not overwrite_cache:
																						logger.info(F'Loading features from cached file {cached_features_file}'      )
																						UpperCAmelCase         =     torch.load(UpperCamelCase__      )
																else:
																						logger.info(F'Creating features from dataset file at {data_dir}'      )

																						UpperCAmelCase         =     (
																						    processor.get_dev_examples(UpperCamelCase__      ) if evaluate else processor.get_train_examples(UpperCamelCase__      )
																						)

																						logger.info("Training examples: %s"				,		len(UpperCamelCase__      )      )
																						UpperCAmelCase         =     hans_convert_examples_to_features(UpperCamelCase__				,		UpperCamelCase__				,		UpperCamelCase__				,		UpperCamelCase__      )
																						logger.info("Saving features into cached file %s"				,		UpperCamelCase__      )
																						torch.save(self.features				,		UpperCamelCase__      )






				def __len__(						self      :     str      )				->					str:
										'''simple docstring'''







										return len(self.features      )






				def __getitem__(						self      :     Tuple				,		UpperCamelCase__      :     Any      )				->					InputFeatures:
										'''simple docstring'''







										return self.features[i]







				def SCREAMING_SNAKE_CASE_			(						self      :     Any      )				->					Tuple:
										'''simple docstring'''







										return self.label_list


if is_tf_available():
		import tensorflow as tf





		class  __magic_name__   :
				lowercase      :       List[InputFeatures]






				def __init__(						self      :     Optional[int]				,		UpperCamelCase__      :     str				,		UpperCamelCase__      :     PreTrainedTokenizer				,		UpperCamelCase__      :     str				,		UpperCamelCase__      :     Optional[int] = 1_28				,		UpperCamelCase__      :     Optional[Any]=False				,		UpperCamelCase__      :     bool = False				,		)				->					List[str]:
										'''simple docstring'''







										UpperCAmelCase         =     hans_processors[task]()
										UpperCAmelCase         =     processor.get_labels()
										if tokenizer.__class__ in (
										    RobertaTokenizer,
										    RobertaTokenizerFast,
										    XLMRobertaTokenizer,
										    BartTokenizer,
										    BartTokenizerFast,
										):
																# HACK(label indices are swapped in RoBERTa pretrained model)
																UpperCAmelCase       ,		UpperCAmelCase         =     label_list[2], label_list[1]
										UpperCAmelCase         =     label_list

										UpperCAmelCase         =     processor.get_dev_examples(UpperCamelCase__      ) if evaluate else processor.get_train_examples(UpperCamelCase__      )
										UpperCAmelCase         =     hans_convert_examples_to_features(UpperCamelCase__				,		UpperCamelCase__				,		UpperCamelCase__				,		UpperCamelCase__      )

										def gen():
																for ex_index, ex in tqdm.tqdm(enumerate(self.features      )				,		desc="convert examples to features"      ):
																						if ex_index % 1_00_00 == 0:
																												logger.info("Writing example %d of %d" % (ex_index, len(UpperCamelCase__      ))      )

																						yield (
																						    {
																						        "example_id": 0,
																						        "input_ids": ex.input_ids,
																						        "attention_mask": ex.attention_mask,
																						        "token_type_ids": ex.token_type_ids,
																						    },
																						    ex.label,
																						)

										UpperCAmelCase         =     tf.data.Dataset.from_generator(
										    UpperCamelCase__				,		(
										        {
										            "example_id": tf.intaa,
										            "input_ids": tf.intaa,
										            "attention_mask": tf.intaa,
										            "token_type_ids": tf.intaa,
										        },
										        tf.intaa,
										    )				,		(
										        {
										            "example_id": tf.TensorShape([]      ),
										            "input_ids": tf.TensorShape([None, None]      ),
										            "attention_mask": tf.TensorShape([None, None]      ),
										            "token_type_ids": tf.TensorShape([None, None]      ),
										        },
										        tf.TensorShape([]      ),
										    )				,		)






				def SCREAMING_SNAKE_CASE_			(						self      :     int      )				->					int:
										'''simple docstring'''







										return self.dataset






				def __len__(						self      :     Any      )				->					Optional[Any]:
										'''simple docstring'''







										return len(self.features      )






				def __getitem__(						self      :     List[Any]				,		UpperCamelCase__      :     List[str]      )				->					InputFeatures:
										'''simple docstring'''







										return self.features[i]







				def SCREAMING_SNAKE_CASE_			(						self      :     str      )				->					Union[str, Any]:
										'''simple docstring'''







										return self.label_list





class  __magic_name__   (							A__						):






		def SCREAMING_SNAKE_CASE_			(						self      :     int				,		UpperCamelCase__      :     List[Any]      )				->					Any:
								'''simple docstring'''







								return self._create_examples(self._read_tsv(os.path.join(UpperCamelCase__				,		"heuristics_train_set.txt"      )      )				,		"train"      )






		def SCREAMING_SNAKE_CASE_			(						self      :     str				,		UpperCamelCase__      :     int      )				->					Dict:
								'''simple docstring'''







								return self._create_examples(self._read_tsv(os.path.join(UpperCamelCase__				,		"heuristics_evaluation_set.txt"      )      )				,		"dev"      )






		def SCREAMING_SNAKE_CASE_			(						self      :     Tuple      )				->					List[Any]:
								'''simple docstring'''







								return ["contradiction", "entailment", "neutral"]







		def SCREAMING_SNAKE_CASE_			(						self      :     int				,		UpperCamelCase__      :     List[str]				,		UpperCamelCase__      :     Optional[Any]      )				->					Dict:
								'''simple docstring'''







								UpperCAmelCase         =     []
								for i, line in enumerate(UpperCamelCase__      ):
														if i == 0:
																				continue
														UpperCAmelCase         =     "%s-%s" % (set_type, line[0])
														UpperCAmelCase         =     line[5]
														UpperCAmelCase         =     line[6]
														UpperCAmelCase         =     line[7][2:] if line[7].startswith("ex"      ) else line[7]
														UpperCAmelCase         =     line[0]
														examples.append(InputExample(guid=UpperCamelCase__				,		text_a=UpperCamelCase__				,		text_b=UpperCamelCase__				,		label=UpperCamelCase__				,		pairID=UpperCamelCase__      )      )
								return examples

def    lowerCamelCase_(lowerCamelCase_    ,   lowerCamelCase_    ,   lowerCamelCase_    ,   lowerCamelCase_    ,   ) ->       List[Any]:
						UpperCAmelCase         =     {label: i for i, label in enumerate(lowerCamelCase_    )}

						UpperCAmelCase         =     []
						for ex_index, example in tqdm.tqdm(enumerate(lowerCamelCase_    )    ,   desc="convert examples to features"    ):
												if ex_index % 10_000 == 0:
																		logger.info("Writing example %d" % (ex_index)    )

												UpperCAmelCase         =     tokenizer(
												    example.text_a    ,   example.text_b    ,   add_special_tokens=lowerCamelCase_    ,   max_length=lowerCamelCase_    ,   padding="max_length"    ,   truncation=lowerCamelCase_    ,   return_overflowing_tokens=lowerCamelCase_    ,   )

												UpperCAmelCase         =     label_map[example.label] if example.label in label_map else 0

												UpperCAmelCase         =     int(example.pairID    )

												features.append(InputFeatures(**lowerCamelCase_    ,   label=lowerCamelCase_    ,   pairID=lowerCamelCase_    )    )

						for i, example in enumerate(examples[:5]    ):
												logger.info("*** Example ***"    )
												logger.info(F'guid: {example}'    )
												logger.info(F'features: {features[i]}'    )

						return features


__lowerCamelCase     :							int  							=				{
    "hans": 3,
}





__lowerCamelCase     :							Optional[int]  							=				{
    "hans": HansProcessor,
}








import os
import re
import warnings
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
		from .tokenization_ta import TaTokenizer
else:
		__lowerCamelCase     :							Optional[int]  							=				None


__lowerCamelCase     :							int  							=				logging.get_logger(__name__)

__lowerCamelCase     :							Any  							=				{"vocab_file": "spiece.model", "tokenizer_file": "tokenizer.json"}

__lowerCamelCase     :							List[str]  							=				{
    "vocab_file": {
        "t5-small": "https://huggingface.co/t5-small/resolve/main/spiece.model",
        "t5-base": "https://huggingface.co/t5-base/resolve/main/spiece.model",
        "t5-large": "https://huggingface.co/t5-large/resolve/main/spiece.model",
        "t5-3b": "https://huggingface.co/t5-3b/resolve/main/spiece.model",
        "t5-11b": "https://huggingface.co/t5-11b/resolve/main/spiece.model",
    },
    "tokenizer_file": {
        "t5-small": "https://huggingface.co/t5-small/resolve/main/tokenizer.json",
        "t5-base": "https://huggingface.co/t5-base/resolve/main/tokenizer.json",
        "t5-large": "https://huggingface.co/t5-large/resolve/main/tokenizer.json",
        "t5-3b": "https://huggingface.co/t5-3b/resolve/main/tokenizer.json",
        "t5-11b": "https://huggingface.co/t5-11b/resolve/main/tokenizer.json",
    },
}


# TODO(PVP) - this should be removed in Transformers v5
__lowerCamelCase     :							List[str]  							=				{
    "t5-small": 512,
    "t5-base": 512,
    "t5-large": 512,
    "t5-3b": 512,
    "t5-11b": 512,
}





class  __magic_name__   (							A__						):
		lowercase      :       List[str]          =VOCAB_FILES_NAMES
		lowercase      :       int          =PRETRAINED_VOCAB_FILES_MAP
		lowercase      :       Union[str, Any]          =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		lowercase      :       Union[str, Any]          =['''input_ids''', '''attention_mask''']
		lowercase      :       Any          =TaTokenizer

		lowercase      :       List[int]          =[]






		def __init__(						self      :     Optional[Any]				,		UpperCamelCase__      :     List[str]=None				,		UpperCamelCase__      :     List[str]=None				,		UpperCamelCase__      :     Any="</s>"				,		UpperCamelCase__      :     Dict="<unk>"				,		UpperCamelCase__      :     Tuple="<pad>"				,		UpperCamelCase__      :     Tuple=1_00				,		UpperCamelCase__      :     List[Any]=None				,		**UpperCamelCase__      :     int				,		)				->					Optional[Any]:
								'''simple docstring'''







								if extra_ids > 0 and additional_special_tokens is None:
														UpperCAmelCase         =     [F'<extra_id_{i}>' for i in range(UpperCamelCase__      )]
								elif extra_ids > 0 and additional_special_tokens is not None:
														# Check that we have the right number of extra special tokens
														UpperCAmelCase         =     len(set(filter(lambda UpperCamelCase__      : bool("extra_id_" in str(UpperCamelCase__      )      )				,		UpperCamelCase__      )      )      )
														if extra_tokens != extra_ids:
																				raise ValueError(
																				    F'Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are'
																				    " provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids"
																				    " tokens"      )

								super().__init__(
								    UpperCamelCase__				,		tokenizer_file=UpperCamelCase__				,		eos_token=UpperCamelCase__				,		unk_token=UpperCamelCase__				,		pad_token=UpperCamelCase__				,		extra_ids=UpperCamelCase__				,		additional_special_tokens=UpperCamelCase__				,		**UpperCamelCase__				,		)

								UpperCAmelCase         =     vocab_file
								UpperCAmelCase         =     False if not self.vocab_file else True
								UpperCAmelCase         =     extra_ids






		@staticmethod
		def SCREAMING_SNAKE_CASE_			(						UpperCamelCase__      :     Optional[int]				,		UpperCamelCase__      :     Dict				,		UpperCamelCase__      :     Union[str, Any]      )				->					Optional[Any]:
								'''simple docstring'''







								if pretrained_model_name_or_path in TaTokenizerFast.max_model_input_sizes:
														UpperCAmelCase         =     TaTokenizerFast.max_model_input_sizes[pretrained_model_name_or_path]
														if init_max_model_length is not None and init_max_model_length != max_model_length:
																				return init_max_model_length
														elif init_max_model_length is None:
																				warnings.warn(
																				    "This tokenizer was incorrectly instantiated with a model max length of"
																				    F' {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this'
																				    " behavior is kept to avoid breaking backwards compatibility when padding/encoding with"
																				    " `truncation is True`.\n- Be aware that you SHOULD NOT rely on"
																				    F' {pretrained_model_name_or_path} automatically truncating your input to'
																				    F' {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences'
																				    F' longer than {deprecated_max_model_length} you can either instantiate this tokenizer with'
																				    " `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please"
																				    " instantiate this tokenizer with `model_max_length` set to your preferred value."				,		UpperCamelCase__				,		)

								return max_model_length






		def SCREAMING_SNAKE_CASE_			(						self      :     Any				,		UpperCamelCase__      :     str				,		UpperCamelCase__      :     Optional[str] = None      )				->					Tuple[str]:
								'''simple docstring'''







								if not self.can_save_slow_tokenizer:
														raise ValueError(
														    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
														    "tokenizer."      )

								if not os.path.isdir(UpperCamelCase__      ):
														logger.error(F'Vocabulary path ({save_directory}) should be a directory'      )
														return
								UpperCAmelCase         =     os.path.join(
								    UpperCamelCase__				,		(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]      )

								if os.path.abspath(self.vocab_file      ) != os.path.abspath(UpperCamelCase__      ):
														copyfile(self.vocab_file				,		UpperCamelCase__      )
														logger.info(F'Copy vocab file to {out_vocab_file}'      )

								return (out_vocab_file,)






		def SCREAMING_SNAKE_CASE_			(						self      :     List[str]				,		UpperCamelCase__      :     List[int]				,		UpperCamelCase__      :     Optional[List[int]] = None      )				->					List[int]:
								'''simple docstring'''







								UpperCAmelCase         =     token_ids_a + [self.eos_token_id]
								if token_ids_a is None:
														return self.prefix_tokens + token_ids_a
								else:
														UpperCAmelCase         =     token_ids_a + [self.eos_token_id]
														return self.prefix_tokens + token_ids_a + token_ids_a






		def SCREAMING_SNAKE_CASE_			(						self      :     Optional[Any]				,		UpperCamelCase__      :     List[int]				,		UpperCamelCase__      :     Optional[List[int]] = None      )				->					List[int]:
								'''simple docstring'''







								UpperCAmelCase         =     [self.eos_token_id]

								if token_ids_a is None:
														return len(token_ids_a + eos      ) * [0]
								return len(token_ids_a + eos + token_ids_a + eos      ) * [0]






		def SCREAMING_SNAKE_CASE_			(						self      :     List[Any]      )				->					List[Any]:
								'''simple docstring'''







								return list(
								    set(filter(lambda UpperCamelCase__      : bool(re.search(R"<extra_id_\d+>"				,		UpperCamelCase__      )      ) is not None				,		self.additional_special_tokens      )      )      )







		def SCREAMING_SNAKE_CASE_			(						self      :     str      )				->					Dict:
								'''simple docstring'''







								return [self.convert_tokens_to_ids(UpperCamelCase__      ) for token in self.get_sentinel_tokens()]



