






'''simple docstring'''







import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


UpperCamelCase    :      List[Any]  			=		logging.getLogger(__name__)




@dataclass(frozen=a			)
class 		UpperCamelCase__    :




		'''simple docstring'''





		_UpperCamelCase				  =					42
		_UpperCamelCase				  =					42
		_UpperCamelCase				  =					None
		_UpperCamelCase				  =					None
		_UpperCamelCase				  =					None




@dataclass(frozen=a			)
class 		UpperCamelCase__    :




		'''simple docstring'''





		_UpperCamelCase				  =					42
		_UpperCamelCase				  =					None
		_UpperCamelCase				  =					None
		_UpperCamelCase				  =					None
		_UpperCamelCase				  =					None


if is_torch_available():
							import torch
							from torch.utils.data import Dataset




							class 		UpperCamelCase__    (a			):




									'''simple docstring'''





									_UpperCamelCase				  =					42






									def __init__(      self   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase = None   ,_lowerCAmelCase=False   ,_lowerCAmelCase = False   ,):
															lowerCamelCase__         =  hans_processors[task]()

															lowerCamelCase__         =  os.path.join(
															    _lowerCAmelCase   ,"""cached_{}_{}_{}_{}""".format(
															        """dev""" if evaluate else """train"""   ,tokenizer.__class__.__name__   ,str(_lowerCAmelCase       )   ,_lowerCAmelCase   ,)   ,)
															lowerCamelCase__         =  processor.get_labels()
															if tokenizer.__class__ in (
															    RobertaTokenizer,
															    RobertaTokenizerFast,
															    XLMRobertaTokenizer,
															    BartTokenizer,
															    BartTokenizerFast,
															):
																					# HACK(label indices are swapped in RoBERTa pretrained model)
																					lowerCamelCase__       ,			lowerCamelCase__         =  label_list[2], label_list[1]
															lowerCamelCase__         =  label_list

															# Make sure only the first process in distributed training processes the dataset,
															# and the others will use the cache.
															lowerCamelCase__         =  cached_features_file + """.lock"""
															with FileLock(_lowerCAmelCase       ):
																					if os.path.exists(_lowerCAmelCase       ) and not overwrite_cache:
																											logger.info(F'''Loading features from cached file {cached_features_file}'''       )
																											lowerCamelCase__         =  torch.load(_lowerCAmelCase       )
																					else:
																											logger.info(F'''Creating features from dataset file at {data_dir}'''       )

																											lowerCamelCase__         =  (
																											    processor.get_dev_examples(_lowerCAmelCase       ) if evaluate else processor.get_train_examples(_lowerCAmelCase       )
																											)

																											logger.info("""Training examples: %s"""   ,len(_lowerCAmelCase       )       )
																											lowerCamelCase__         =  hans_convert_examples_to_features(_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase       )
																											logger.info("""Saving features into cached file %s"""   ,_lowerCAmelCase       )
																											torch.save(self.features   ,_lowerCAmelCase       )






									def __len__(      self       ):
															return len(self.features       )






									def __getitem__(      self   ,_lowerCAmelCase       ):
															return self.features[i]






									def 			UpperCamelCase_					(      self       ):
															return self.label_list


if is_tf_available():
							import tensorflow as tf




							class 		UpperCamelCase__    :




									'''simple docstring'''





									_UpperCamelCase				  =					42






									def __init__(      self   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase = 1_28   ,_lowerCAmelCase=False   ,_lowerCAmelCase = False   ,):
															lowerCamelCase__         =  hans_processors[task]()
															lowerCamelCase__         =  processor.get_labels()
															if tokenizer.__class__ in (
															    RobertaTokenizer,
															    RobertaTokenizerFast,
															    XLMRobertaTokenizer,
															    BartTokenizer,
															    BartTokenizerFast,
															):
																					# HACK(label indices are swapped in RoBERTa pretrained model)
																					lowerCamelCase__       ,			lowerCamelCase__         =  label_list[2], label_list[1]
															lowerCamelCase__         =  label_list

															lowerCamelCase__         =  processor.get_dev_examples(_lowerCAmelCase       ) if evaluate else processor.get_train_examples(_lowerCAmelCase       )
															lowerCamelCase__         =  hans_convert_examples_to_features(_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase       )

															def gen():
																					for ex_index, ex in tqdm.tqdm(enumerate(self.features       )   ,desc="""convert examples to features"""       ):
																											if ex_index % 1_00_00 == 0:
																																	logger.info("""Writing example %d of %d""" % (ex_index, len(_lowerCAmelCase       ))       )

																											yield (
																											    {
																											        "example_id": 0,
																											        "input_ids": ex.input_ids,
																											        "attention_mask": ex.attention_mask,
																											        "token_type_ids": ex.token_type_ids,
																											    },
																											    ex.label,
																											)

															lowerCamelCase__         =  tf.data.Dataset.from_generator(
															    _lowerCAmelCase   ,(
															        {
															            """example_id""": tf.intaa,
															            """input_ids""": tf.intaa,
															            """attention_mask""": tf.intaa,
															            """token_type_ids""": tf.intaa,
															        },
															        tf.intaa,
															    )   ,(
															        {
															            """example_id""": tf.TensorShape([]       ),
															            """input_ids""": tf.TensorShape([None, None]       ),
															            """attention_mask""": tf.TensorShape([None, None]       ),
															            """token_type_ids""": tf.TensorShape([None, None]       ),
															        },
															        tf.TensorShape([]       ),
															    )   ,)






									def 			UpperCamelCase_					(      self       ):
															return self.dataset






									def __len__(      self       ):
															return len(self.features       )






									def __getitem__(      self   ,_lowerCAmelCase       ):
															return self.features[i]






									def 			UpperCamelCase_					(      self       ):
															return self.label_list




class 		UpperCamelCase__    (a			):




		'''simple docstring'''





		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase       ):
								return self._create_examples(self._read_tsv(os.path.join(_lowerCAmelCase   ,"""heuristics_train_set.txt"""       )       )   ,"""train"""       )






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase       ):
								return self._create_examples(self._read_tsv(os.path.join(_lowerCAmelCase   ,"""heuristics_evaluation_set.txt"""       )       )   ,"""dev"""       )






		def 			UpperCamelCase_					(      self       ):
								return ["contradiction", "entailment", "neutral"]






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase   ,_lowerCAmelCase       ):
								lowerCamelCase__         =  []
								for i, line in enumerate(_lowerCAmelCase       ):
														if i == 0:
																				continue
														lowerCamelCase__         =  """%s-%s""" % (set_type, line[0])
														lowerCamelCase__         =  line[5]
														lowerCamelCase__         =  line[6]
														lowerCamelCase__         =  line[7][2:] if line[7].startswith("""ex"""       ) else line[7]
														lowerCamelCase__         =  line[0]
														examples.append(InputExample(guid=_lowerCAmelCase   ,text_a=_lowerCAmelCase   ,text_b=_lowerCAmelCase   ,label=_lowerCAmelCase   ,pairID=_lowerCAmelCase       )       )
								return examples



def 						A__			(				__lowerCAmelCase		:			List[InputExample]   ,    __lowerCAmelCase		:			List[str]   ,    __lowerCAmelCase		:			int   ,    __lowerCAmelCase		:			PreTrainedTokenizer   ,    ):
						lowerCamelCase__         =  {label: i for i, label in enumerate(__lowerCAmelCase						)}

						lowerCamelCase__         =  []
						for ex_index, example in tqdm.tqdm(enumerate(__lowerCAmelCase						)   ,    desc="""convert examples to features"""						):
												if ex_index % 1_0000 == 0:
																		logger.info("""Writing example %d""" % (ex_index)						)

												lowerCamelCase__         =  tokenizer(
												    example.text_a   ,    example.text_b   ,    add_special_tokens=__lowerCAmelCase   ,    max_length=__lowerCAmelCase   ,    padding="""max_length"""   ,    truncation=__lowerCAmelCase   ,    return_overflowing_tokens=__lowerCAmelCase   ,    )

												lowerCamelCase__         =  label_map[example.label] if example.label in label_map else 0

												lowerCamelCase__         =  int(example.pairID						)

												features.append(InputFeatures(**__lowerCAmelCase   ,    label=__lowerCAmelCase   ,    pairID=__lowerCAmelCase						)						)

						for i, example in enumerate(examples[:5]						):
												logger.info("""*** Example ***"""						)
												logger.info(F'''guid: {example}'''						)
												logger.info(F'''features: {features[i]}'''						)

						return features


UpperCamelCase    :      List[Any]  			=		{
    'hans': 3,
}





UpperCamelCase    :      Tuple  			=		{
    'hans': HansProcessor,
}









'''simple docstring'''







import os
import tempfile
import unittest
from pathlib import Path

from transformers import AutoConfig, is_tf_available
from transformers.testing_utils import require_tf


if is_tf_available():
							import tensorflow as tf

							from transformers import TensorFlowBenchmark, TensorFlowBenchmarkArguments




@require_tf
class 		UpperCamelCase__    (unittest.TestCase			):




		'''simple docstring'''





		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase       ):
								for model_result in results.values():
														for batch_size, sequence_length in zip(model_result["""bs"""]   ,model_result["""ss"""]       ):
																				lowerCamelCase__         =  model_result["""result"""][batch_size][sequence_length]
																				self.assertIsNotNone(_lowerCAmelCase       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  """sshleifer/tiny-gpt2"""
								lowerCamelCase__         =  TensorFlowBenchmarkArguments(
								    models=[MODEL_ID]   ,training=_lowerCAmelCase   ,inference=_lowerCAmelCase   ,sequence_lengths=[8]   ,batch_sizes=[1]   ,eager_mode=_lowerCAmelCase   ,multi_process=_lowerCAmelCase   ,)
								lowerCamelCase__         =  TensorFlowBenchmark(_lowerCAmelCase       )
								lowerCamelCase__         =  benchmark.run()
								self.check_results_dict_not_empty(results.time_inference_result       )
								self.check_results_dict_not_empty(results.memory_inference_result       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  """sgugger/tiny-distilbert-classification"""
								lowerCamelCase__         =  TensorFlowBenchmarkArguments(
								    models=[MODEL_ID]   ,training=_lowerCAmelCase   ,inference=_lowerCAmelCase   ,sequence_lengths=[8]   ,batch_sizes=[1]   ,multi_process=_lowerCAmelCase   ,only_pretrain_model=_lowerCAmelCase   ,)
								lowerCamelCase__         =  TensorFlowBenchmark(_lowerCAmelCase       )
								lowerCamelCase__         =  benchmark.run()
								self.check_results_dict_not_empty(results.time_inference_result       )
								self.check_results_dict_not_empty(results.memory_inference_result       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  """sshleifer/tiny-gpt2"""
								lowerCamelCase__         =  TensorFlowBenchmarkArguments(
								    models=[MODEL_ID]   ,training=_lowerCAmelCase   ,inference=_lowerCAmelCase   ,sequence_lengths=[8]   ,batch_sizes=[1]   ,multi_process=_lowerCAmelCase   ,)
								lowerCamelCase__         =  TensorFlowBenchmark(_lowerCAmelCase       )
								lowerCamelCase__         =  benchmark.run()
								self.check_results_dict_not_empty(results.time_inference_result       )
								self.check_results_dict_not_empty(results.memory_inference_result       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  """sshleifer/tiny-gpt2"""
								lowerCamelCase__         =  AutoConfig.from_pretrained(_lowerCAmelCase       )
								lowerCamelCase__         =  TensorFlowBenchmarkArguments(
								    models=[MODEL_ID]   ,training=_lowerCAmelCase   ,inference=_lowerCAmelCase   ,sequence_lengths=[8]   ,batch_sizes=[1]   ,eager_mode=_lowerCAmelCase   ,multi_process=_lowerCAmelCase   ,)
								lowerCamelCase__         =  TensorFlowBenchmark(_lowerCAmelCase   ,[config]       )
								lowerCamelCase__         =  benchmark.run()
								self.check_results_dict_not_empty(results.time_inference_result       )
								self.check_results_dict_not_empty(results.memory_inference_result       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  """sshleifer/tiny-gpt2"""
								lowerCamelCase__         =  AutoConfig.from_pretrained(_lowerCAmelCase       )
								lowerCamelCase__         =  TensorFlowBenchmarkArguments(
								    models=[MODEL_ID]   ,training=_lowerCAmelCase   ,inference=_lowerCAmelCase   ,sequence_lengths=[8]   ,batch_sizes=[1]   ,multi_process=_lowerCAmelCase   ,)
								lowerCamelCase__         =  TensorFlowBenchmark(_lowerCAmelCase   ,[config]       )
								lowerCamelCase__         =  benchmark.run()
								self.check_results_dict_not_empty(results.time_inference_result       )
								self.check_results_dict_not_empty(results.memory_inference_result       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  """sshleifer/tiny-gpt2"""
								lowerCamelCase__         =  TensorFlowBenchmarkArguments(
								    models=[MODEL_ID]   ,training=_lowerCAmelCase   ,inference=_lowerCAmelCase   ,sequence_lengths=[8]   ,batch_sizes=[1]   ,multi_process=_lowerCAmelCase   ,)
								lowerCamelCase__         =  TensorFlowBenchmark(_lowerCAmelCase       )
								lowerCamelCase__         =  benchmark.run()
								self.check_results_dict_not_empty(results.time_train_result       )
								self.check_results_dict_not_empty(results.memory_train_result       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  """sshleifer/tiny-gpt2"""
								lowerCamelCase__         =  AutoConfig.from_pretrained(_lowerCAmelCase       )
								lowerCamelCase__         =  TensorFlowBenchmarkArguments(
								    models=[MODEL_ID]   ,training=_lowerCAmelCase   ,inference=_lowerCAmelCase   ,sequence_lengths=[8]   ,batch_sizes=[1]   ,multi_process=_lowerCAmelCase   ,)
								lowerCamelCase__         =  TensorFlowBenchmark(_lowerCAmelCase   ,[config]       )
								lowerCamelCase__         =  benchmark.run()
								self.check_results_dict_not_empty(results.time_train_result       )
								self.check_results_dict_not_empty(results.memory_train_result       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  """patrickvonplaten/t5-tiny-random"""
								lowerCamelCase__         =  AutoConfig.from_pretrained(_lowerCAmelCase       )
								lowerCamelCase__         =  TensorFlowBenchmarkArguments(
								    models=[MODEL_ID]   ,training=_lowerCAmelCase   ,inference=_lowerCAmelCase   ,sequence_lengths=[8]   ,batch_sizes=[1]   ,multi_process=_lowerCAmelCase   ,)
								lowerCamelCase__         =  TensorFlowBenchmark(_lowerCAmelCase   ,configs=[config]       )
								lowerCamelCase__         =  benchmark.run()
								self.check_results_dict_not_empty(results.time_inference_result       )
								self.check_results_dict_not_empty(results.memory_inference_result       )






		@unittest.skipIf(is_tf_available() and len(tf.config.list_physical_devices("""GPU"""       )       ) == 0   ,"""Cannot do xla on CPU."""       )
		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  """sshleifer/tiny-gpt2"""
								lowerCamelCase__         =  TensorFlowBenchmarkArguments(
								    models=[MODEL_ID]   ,training=_lowerCAmelCase   ,inference=_lowerCAmelCase   ,sequence_lengths=[8]   ,batch_sizes=[1]   ,use_xla=_lowerCAmelCase   ,multi_process=_lowerCAmelCase   ,)
								lowerCamelCase__         =  TensorFlowBenchmark(_lowerCAmelCase       )
								lowerCamelCase__         =  benchmark.run()
								self.check_results_dict_not_empty(results.time_inference_result       )
								self.check_results_dict_not_empty(results.memory_inference_result       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  """sshleifer/tiny-gpt2"""
								with tempfile.TemporaryDirectory() as tmp_dir:
														lowerCamelCase__         =  TensorFlowBenchmarkArguments(
														    models=[MODEL_ID]   ,inference=_lowerCAmelCase   ,save_to_csv=_lowerCAmelCase   ,sequence_lengths=[8]   ,batch_sizes=[1]   ,inference_time_csv_file=os.path.join(_lowerCAmelCase   ,"""inf_time.csv"""       )   ,inference_memory_csv_file=os.path.join(_lowerCAmelCase   ,"""inf_mem.csv"""       )   ,env_info_csv_file=os.path.join(_lowerCAmelCase   ,"""env.csv"""       )   ,multi_process=_lowerCAmelCase   ,)
														lowerCamelCase__         =  TensorFlowBenchmark(_lowerCAmelCase       )
														benchmark.run()
														self.assertTrue(Path(os.path.join(_lowerCAmelCase   ,"""inf_time.csv"""       )       ).exists()       )
														self.assertTrue(Path(os.path.join(_lowerCAmelCase   ,"""inf_mem.csv"""       )       ).exists()       )
														self.assertTrue(Path(os.path.join(_lowerCAmelCase   ,"""env.csv"""       )       ).exists()       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  """sshleifer/tiny-gpt2"""

								def _check_summary_is_not_empty(_lowerCAmelCase       ):
														self.assertTrue(hasattr(_lowerCAmelCase   ,"""sequential"""       )       )
														self.assertTrue(hasattr(_lowerCAmelCase   ,"""cumulative"""       )       )
														self.assertTrue(hasattr(_lowerCAmelCase   ,"""current"""       )       )
														self.assertTrue(hasattr(_lowerCAmelCase   ,"""total"""       )       )

								with tempfile.TemporaryDirectory() as tmp_dir:
														lowerCamelCase__         =  TensorFlowBenchmarkArguments(
														    models=[MODEL_ID]   ,inference=_lowerCAmelCase   ,sequence_lengths=[8]   ,batch_sizes=[1]   ,log_filename=os.path.join(_lowerCAmelCase   ,"""log.txt"""       )   ,log_print=_lowerCAmelCase   ,trace_memory_line_by_line=_lowerCAmelCase   ,eager_mode=_lowerCAmelCase   ,multi_process=_lowerCAmelCase   ,)
														lowerCamelCase__         =  TensorFlowBenchmark(_lowerCAmelCase       )
														lowerCamelCase__         =  benchmark.run()
														_check_summary_is_not_empty(result.inference_summary       )
														self.assertTrue(Path(os.path.join(_lowerCAmelCase   ,"""log.txt"""       )       ).exists()       )

