



import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


SCREAMING_SNAKE_CASE     				=						logging.getLogger(__name__)



@dataclass(frozen=lowercase__  )
class 	lowerCamelCase      :







					'''simple docstring'''







					lowerCAmelCase_				:    str
					lowerCAmelCase_				:    str
					lowerCAmelCase_				:    Optional[str]									=  None
					lowerCAmelCase_				:    Optional[str]									=  None
					lowerCAmelCase_				:    Optional[str]									=  None





@dataclass(frozen=lowercase__  )
class 	lowerCamelCase      :







					'''simple docstring'''







					lowerCAmelCase_				:    List[int]
					lowerCAmelCase_				:    Optional[List[int]]									=  None
					lowerCAmelCase_				:    Optional[List[int]]									=  None
					lowerCAmelCase_				:    Optional[Union[int, float]]									=  None
					lowerCAmelCase_				:    Optional[int]									=  None


if is_torch_available():
			import torch
			from torch.utils.data import Dataset



			class 	lowerCamelCase      (						lowercase__  ):







								'''simple docstring'''







								lowerCAmelCase_				:    List[InputFeatures]



								def __init__(			self    ,	lowerCAmelCase    ,	lowerCAmelCase    ,	lowerCAmelCase    ,	lowerCAmelCase = None    ,	lowerCAmelCase=False    ,	lowerCAmelCase = False    ,	):
										UpperCAmelCase_						=				hans_processors[task]()

										UpperCAmelCase_						=				os.path.join(
										    lowerCAmelCase    ,	"cached_{}_{}_{}_{}".format(
										        "dev" if evaluate else "train"    ,	tokenizer.__class__.__name__    ,	str(lowerCAmelCase     )    ,	lowerCAmelCase    ,	)    ,	)
										UpperCAmelCase_						=				processor.get_labels()
										if tokenizer.__class__ in (
										    RobertaTokenizer,
										    RobertaTokenizerFast,
										    XLMRobertaTokenizer,
										    BartTokenizer,
										    BartTokenizerFast,
										):
												# HACK(label indices are swapped in RoBERTa pretrained model)
												UpperCAmelCase_					,						UpperCAmelCase_						=				label_list[2], label_list[1]
										UpperCAmelCase_						=				label_list

										# Make sure only the first process in distributed training processes the dataset,
										# and the others will use the cache.
										UpperCAmelCase_						=				cached_features_file + ".lock"
										with FileLock(lowerCAmelCase     ):
												if os.path.exists(lowerCAmelCase     ) and not overwrite_cache:
														logger.info(f'''Loading features from cached file {cached_features_file}'''     )
														UpperCAmelCase_						=				torch.load(lowerCAmelCase     )
												else:
														logger.info(f'''Creating features from dataset file at {data_dir}'''     )

														UpperCAmelCase_						=				(
														    processor.get_dev_examples(lowerCAmelCase     ) if evaluate else processor.get_train_examples(lowerCAmelCase     )
														)

														logger.info("Training examples: %s"    ,	len(lowerCAmelCase     )     )
														UpperCAmelCase_						=				hans_convert_examples_to_features(lowerCAmelCase    ,	lowerCAmelCase    ,	lowerCAmelCase    ,	lowerCAmelCase     )
														logger.info("Saving features into cached file %s"    ,	lowerCAmelCase     )
														torch.save(self.features    ,	lowerCAmelCase     )



								def __len__(			self     ):
										return len(self.features     )



								def __getitem__(			self    ,	lowerCAmelCase     ):
										return self.features[i]



								def    A__		(			self     ):
										return self.label_list


if is_tf_available():
			import tensorflow as tf



			class 	lowerCamelCase      :







								'''simple docstring'''







								lowerCAmelCase_				:    List[InputFeatures]



								def __init__(			self    ,	lowerCAmelCase    ,	lowerCAmelCase    ,	lowerCAmelCase    ,	lowerCAmelCase = 128    ,	lowerCAmelCase=False    ,	lowerCAmelCase = False    ,	):
										UpperCAmelCase_						=				hans_processors[task]()
										UpperCAmelCase_						=				processor.get_labels()
										if tokenizer.__class__ in (
										    RobertaTokenizer,
										    RobertaTokenizerFast,
										    XLMRobertaTokenizer,
										    BartTokenizer,
										    BartTokenizerFast,
										):
												# HACK(label indices are swapped in RoBERTa pretrained model)
												UpperCAmelCase_					,						UpperCAmelCase_						=				label_list[2], label_list[1]
										UpperCAmelCase_						=				label_list

										UpperCAmelCase_						=				processor.get_dev_examples(lowerCAmelCase     ) if evaluate else processor.get_train_examples(lowerCAmelCase     )
										UpperCAmelCase_						=				hans_convert_examples_to_features(lowerCAmelCase    ,	lowerCAmelCase    ,	lowerCAmelCase    ,	lowerCAmelCase     )

										def gen():
												for ex_index, ex in tqdm.tqdm(enumerate(self.features     )    ,	desc="convert examples to features"     ):
														if ex_index % 1_0000 == 0:
																logger.info("Writing example %d of %d" % (ex_index, len(lowerCAmelCase     ))     )

														yield (
														    {
														        "example_id": 0,
														        "input_ids": ex.input_ids,
														        "attention_mask": ex.attention_mask,
														        "token_type_ids": ex.token_type_ids,
														    },
														    ex.label,
														)

										UpperCAmelCase_						=				tf.data.Dataset.from_generator(
										    lowerCAmelCase    ,	(
										        {
										            "example_id": tf.intaa,
										            "input_ids": tf.intaa,
										            "attention_mask": tf.intaa,
										            "token_type_ids": tf.intaa,
										        },
										        tf.intaa,
										    )    ,	(
										        {
										            "example_id": tf.TensorShape([]     ),
										            "input_ids": tf.TensorShape([None, None]     ),
										            "attention_mask": tf.TensorShape([None, None]     ),
										            "token_type_ids": tf.TensorShape([None, None]     ),
										        },
										        tf.TensorShape([]     ),
										    )    ,	)



								def    A__		(			self     ):
										return self.dataset



								def __len__(			self     ):
										return len(self.features     )



								def __getitem__(			self    ,	lowerCAmelCase     ):
										return self.features[i]



								def    A__		(			self     ):
										return self.label_list



class 	lowerCamelCase      (						lowercase__  ):







					'''simple docstring'''







					def    A__		(			self    ,	lowerCAmelCase     ):
							return self._create_examples(self._read_tsv(os.path.join(lowerCAmelCase    ,	"heuristics_train_set.txt"     )     )    ,	"train"     )



					def    A__		(			self    ,	lowerCAmelCase     ):
							return self._create_examples(self._read_tsv(os.path.join(lowerCAmelCase    ,	"heuristics_evaluation_set.txt"     )     )    ,	"dev"     )



					def    A__		(			self     ):
							return ["contradiction", "entailment", "neutral"]



					def    A__		(			self    ,	lowerCAmelCase    ,	lowerCAmelCase     ):
							UpperCAmelCase_						=				[]
							for i, line in enumerate(lowerCAmelCase     ):
									if i == 0:
											continue
									UpperCAmelCase_						=				"%s-%s" % (set_type, line[0])
									UpperCAmelCase_						=				line[5]
									UpperCAmelCase_						=				line[6]
									UpperCAmelCase_						=				line[7][2:] if line[7].startswith("ex"     ) else line[7]
									UpperCAmelCase_						=				line[0]
									examples.append(InputExample(guid=lowerCAmelCase    ,	text_a=lowerCAmelCase    ,	text_b=lowerCAmelCase    ,	label=lowerCAmelCase    ,	pairID=lowerCAmelCase     )     )
							return examples







def snake_case__  (			__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     )		->							int:
		UpperCAmelCase_						=				{label: i for i, label in enumerate(__SCREAMING_SNAKE_CASE )}

		UpperCAmelCase_						=				[]
		for ex_index, example in tqdm.tqdm(enumerate(__SCREAMING_SNAKE_CASE )			,     desc="convert examples to features" ):
				if ex_index % 1_0000 == 0:
						logger.info("Writing example %d" % (ex_index) )

				UpperCAmelCase_						=				tokenizer(
				    example.text_a			,     example.text_b			,     add_special_tokens=__SCREAMING_SNAKE_CASE			,     max_length=__SCREAMING_SNAKE_CASE			,     padding="max_length"			,     truncation=__SCREAMING_SNAKE_CASE			,     return_overflowing_tokens=__SCREAMING_SNAKE_CASE			,     )

				UpperCAmelCase_						=				label_map[example.label] if example.label in label_map else 0

				UpperCAmelCase_						=				int(example.pairID )

				features.append(InputFeatures(**__SCREAMING_SNAKE_CASE			,     label=__SCREAMING_SNAKE_CASE			,     pairID=__SCREAMING_SNAKE_CASE ) )

		for i, example in enumerate(examples[:5] ):
				logger.info("*** Example ***" )
				logger.info(f'''guid: {example}''' )
				logger.info(f'''features: {features[i]}''' )

		return features


SCREAMING_SNAKE_CASE     				=						{
    "hans": 3,
}

SCREAMING_SNAKE_CASE     				=						{
    "hans": HansProcessor,
}






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


SCREAMING_SNAKE_CASE     				=						logging.get_logger(__name__)

SCREAMING_SNAKE_CASE     				=						{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

SCREAMING_SNAKE_CASE     				=						{
    "vocab_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-german-cased": "https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt",
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-german-cased": (
            "https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json"
        ),
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
    },
}

SCREAMING_SNAKE_CASE     				=						{
    "distilbert-base-uncased": 512,
    "distilbert-base-uncased-distilled-squad": 512,
    "distilbert-base-cased": 512,
    "distilbert-base-cased-distilled-squad": 512,
    "distilbert-base-german-cased": 512,
    "distilbert-base-multilingual-cased": 512,
}


SCREAMING_SNAKE_CASE     				=						{
    "distilbert-base-uncased": {"do_lower_case": True},
    "distilbert-base-uncased-distilled-squad": {"do_lower_case": True},
    "distilbert-base-cased": {"do_lower_case": False},
    "distilbert-base-cased-distilled-squad": {"do_lower_case": False},
    "distilbert-base-german-cased": {"do_lower_case": False},
    "distilbert-base-multilingual-cased": {"do_lower_case": False},
}



class 	lowerCamelCase      (						lowercase__  ):







					'''simple docstring'''







					lowerCAmelCase_				:    Any									=  VOCAB_FILES_NAMES
					lowerCAmelCase_				:    List[str]									=  PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase_				:    str									=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase_				:    Union[str, Any]									=  PRETRAINED_INIT_CONFIGURATION
					lowerCAmelCase_				:    int									=  ['input_ids', 'attention_mask']
					lowerCAmelCase_				:    str									=  DistilBertTokenizer



					def __init__(			self    ,	lowerCAmelCase=None    ,	lowerCAmelCase=None    ,	lowerCAmelCase=True    ,	lowerCAmelCase="[UNK]"    ,	lowerCAmelCase="[SEP]"    ,	lowerCAmelCase="[PAD]"    ,	lowerCAmelCase="[CLS]"    ,	lowerCAmelCase="[MASK]"    ,	lowerCAmelCase=True    ,	lowerCAmelCase=None    ,	**lowerCAmelCase    ,	):
							super().__init__(
							    lowerCAmelCase    ,	tokenizer_file=lowerCAmelCase    ,	do_lower_case=lowerCAmelCase    ,	unk_token=lowerCAmelCase    ,	sep_token=lowerCAmelCase    ,	pad_token=lowerCAmelCase    ,	cls_token=lowerCAmelCase    ,	mask_token=lowerCAmelCase    ,	tokenize_chinese_chars=lowerCAmelCase    ,	strip_accents=lowerCAmelCase    ,	**lowerCAmelCase    ,	)

							UpperCAmelCase_						=				json.loads(self.backend_tokenizer.normalizer.__getstate__()     )
							if (
							    normalizer_state.get("lowercase"    ,	lowerCAmelCase     ) != do_lower_case
							    or normalizer_state.get("strip_accents"    ,	lowerCAmelCase     ) != strip_accents
							    or normalizer_state.get("handle_chinese_chars"    ,	lowerCAmelCase     ) != tokenize_chinese_chars
							):
									UpperCAmelCase_						=				getattr(lowerCAmelCase    ,	normalizer_state.pop("type"     )     )
									UpperCAmelCase_						=				do_lower_case
									UpperCAmelCase_						=				strip_accents
									UpperCAmelCase_						=				tokenize_chinese_chars
									UpperCAmelCase_						=				normalizer_class(**lowerCAmelCase     )

							UpperCAmelCase_						=				do_lower_case



					def    A__		(			self    ,	lowerCAmelCase    ,	lowerCAmelCase=None     ):
							UpperCAmelCase_						=				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

							if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

							return output



					def    A__		(			self    ,	lowerCAmelCase    ,	lowerCAmelCase = None     ):
							UpperCAmelCase_						=				[self.sep_token_id]
							UpperCAmelCase_						=				[self.cls_token_id]
							if token_ids_a is None:
									return len(cls + token_ids_a + sep     ) * [0]
							return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]



					def    A__		(			self    ,	lowerCAmelCase    ,	lowerCAmelCase = None     ):
							UpperCAmelCase_						=				self._tokenizer.model.save(lowerCAmelCase    ,	name=lowerCAmelCase     )
							return tuple(lowerCAmelCase     )

