






'''simple docstring'''







import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


UpperCamelCase    :      List[Any]  			=		logging.getLogger(__name__)




@dataclass(frozen=a			)
class 		UpperCamelCase__    :




		'''simple docstring'''





		_UpperCamelCase				  =					42
		_UpperCamelCase				  =					42
		_UpperCamelCase				  =					None
		_UpperCamelCase				  =					None
		_UpperCamelCase				  =					None




@dataclass(frozen=a			)
class 		UpperCamelCase__    :




		'''simple docstring'''





		_UpperCamelCase				  =					42
		_UpperCamelCase				  =					None
		_UpperCamelCase				  =					None
		_UpperCamelCase				  =					None
		_UpperCamelCase				  =					None


if is_torch_available():
			import torch
			from torch.utils.data import Dataset




			class 		UpperCamelCase__    (a			):




					'''simple docstring'''





					_UpperCamelCase				  =					42






					def __init__(      self   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase = None   ,_lowerCAmelCase=False   ,_lowerCAmelCase = False   ,):
											lowerCamelCase__         =  hans_processors[task]()

											lowerCamelCase__         =  os.path.join(
											    _lowerCAmelCase   ,"""cached_{}_{}_{}_{}""".format(
											        """dev""" if evaluate else """train"""   ,tokenizer.__class__.__name__   ,str(_lowerCAmelCase       )   ,_lowerCAmelCase   ,)   ,)
											lowerCamelCase__         =  processor.get_labels()
											if tokenizer.__class__ in (
											    RobertaTokenizer,
											    RobertaTokenizerFast,
											    XLMRobertaTokenizer,
											    BartTokenizer,
											    BartTokenizerFast,
											):
																	# HACK(label indices are swapped in RoBERTa pretrained model)
																	lowerCamelCase__       ,			lowerCamelCase__         =  label_list[2], label_list[1]
											lowerCamelCase__         =  label_list

											# Make sure only the first process in distributed training processes the dataset,
											# and the others will use the cache.
											lowerCamelCase__         =  cached_features_file + """.lock"""
											with FileLock(_lowerCAmelCase       ):
																	if os.path.exists(_lowerCAmelCase       ) and not overwrite_cache:
																							logger.info(F'''Loading features from cached file {cached_features_file}'''       )
																							lowerCamelCase__         =  torch.load(_lowerCAmelCase       )
																	else:
																							logger.info(F'''Creating features from dataset file at {data_dir}'''       )

																							lowerCamelCase__         =  (
																							    processor.get_dev_examples(_lowerCAmelCase       ) if evaluate else processor.get_train_examples(_lowerCAmelCase       )
																							)

																							logger.info("""Training examples: %s"""   ,len(_lowerCAmelCase       )       )
																							lowerCamelCase__         =  hans_convert_examples_to_features(_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase       )
																							logger.info("""Saving features into cached file %s"""   ,_lowerCAmelCase       )
																							torch.save(self.features   ,_lowerCAmelCase       )






					def __len__(      self       ):
											return len(self.features       )






					def __getitem__(      self   ,_lowerCAmelCase       ):
											return self.features[i]






					def 			UpperCamelCase_					(      self       ):
											return self.label_list


if is_tf_available():
			import tensorflow as tf




			class 		UpperCamelCase__    :




					'''simple docstring'''





					_UpperCamelCase				  =					42






					def __init__(      self   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase = 1_28   ,_lowerCAmelCase=False   ,_lowerCAmelCase = False   ,):
											lowerCamelCase__         =  hans_processors[task]()
											lowerCamelCase__         =  processor.get_labels()
											if tokenizer.__class__ in (
											    RobertaTokenizer,
											    RobertaTokenizerFast,
											    XLMRobertaTokenizer,
											    BartTokenizer,
											    BartTokenizerFast,
											):
																	# HACK(label indices are swapped in RoBERTa pretrained model)
																	lowerCamelCase__       ,			lowerCamelCase__         =  label_list[2], label_list[1]
											lowerCamelCase__         =  label_list

											lowerCamelCase__         =  processor.get_dev_examples(_lowerCAmelCase       ) if evaluate else processor.get_train_examples(_lowerCAmelCase       )
											lowerCamelCase__         =  hans_convert_examples_to_features(_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase       )

											def gen():
																	for ex_index, ex in tqdm.tqdm(enumerate(self.features       )   ,desc="""convert examples to features"""       ):
																							if ex_index % 1_00_00 == 0:
																													logger.info("""Writing example %d of %d""" % (ex_index, len(_lowerCAmelCase       ))       )

																							yield (
																							    {
																							        "example_id": 0,
																							        "input_ids": ex.input_ids,
																							        "attention_mask": ex.attention_mask,
																							        "token_type_ids": ex.token_type_ids,
																							    },
																							    ex.label,
																							)

											lowerCamelCase__         =  tf.data.Dataset.from_generator(
											    _lowerCAmelCase   ,(
											        {
											            """example_id""": tf.intaa,
											            """input_ids""": tf.intaa,
											            """attention_mask""": tf.intaa,
											            """token_type_ids""": tf.intaa,
											        },
											        tf.intaa,
											    )   ,(
											        {
											            """example_id""": tf.TensorShape([]       ),
											            """input_ids""": tf.TensorShape([None, None]       ),
											            """attention_mask""": tf.TensorShape([None, None]       ),
											            """token_type_ids""": tf.TensorShape([None, None]       ),
											        },
											        tf.TensorShape([]       ),
											    )   ,)






					def 			UpperCamelCase_					(      self       ):
											return self.dataset






					def __len__(      self       ):
											return len(self.features       )






					def __getitem__(      self   ,_lowerCAmelCase       ):
											return self.features[i]






					def 			UpperCamelCase_					(      self       ):
											return self.label_list




class 		UpperCamelCase__    (a			):




		'''simple docstring'''





		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase       ):
								return self._create_examples(self._read_tsv(os.path.join(_lowerCAmelCase   ,"""heuristics_train_set.txt"""       )       )   ,"""train"""       )






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase       ):
								return self._create_examples(self._read_tsv(os.path.join(_lowerCAmelCase   ,"""heuristics_evaluation_set.txt"""       )       )   ,"""dev"""       )






		def 			UpperCamelCase_					(      self       ):
								return ["contradiction", "entailment", "neutral"]






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase   ,_lowerCAmelCase       ):
								lowerCamelCase__         =  []
								for i, line in enumerate(_lowerCAmelCase       ):
														if i == 0:
																				continue
														lowerCamelCase__         =  """%s-%s""" % (set_type, line[0])
														lowerCamelCase__         =  line[5]
														lowerCamelCase__         =  line[6]
														lowerCamelCase__         =  line[7][2:] if line[7].startswith("""ex"""       ) else line[7]
														lowerCamelCase__         =  line[0]
														examples.append(InputExample(guid=_lowerCAmelCase   ,text_a=_lowerCAmelCase   ,text_b=_lowerCAmelCase   ,label=_lowerCAmelCase   ,pairID=_lowerCAmelCase       )       )
								return examples



def 						A__			(				__lowerCAmelCase		:			List[InputExample]   ,    __lowerCAmelCase		:			List[str]   ,    __lowerCAmelCase		:			int   ,    __lowerCAmelCase		:			PreTrainedTokenizer   ,    ):
						lowerCamelCase__         =  {label: i for i, label in enumerate(__lowerCAmelCase						)}

						lowerCamelCase__         =  []
						for ex_index, example in tqdm.tqdm(enumerate(__lowerCAmelCase						)   ,    desc="""convert examples to features"""						):
												if ex_index % 1_0000 == 0:
																		logger.info("""Writing example %d""" % (ex_index)						)

												lowerCamelCase__         =  tokenizer(
												    example.text_a   ,    example.text_b   ,    add_special_tokens=__lowerCAmelCase   ,    max_length=__lowerCAmelCase   ,    padding="""max_length"""   ,    truncation=__lowerCAmelCase   ,    return_overflowing_tokens=__lowerCAmelCase   ,    )

												lowerCamelCase__         =  label_map[example.label] if example.label in label_map else 0

												lowerCamelCase__         =  int(example.pairID						)

												features.append(InputFeatures(**__lowerCAmelCase   ,    label=__lowerCAmelCase   ,    pairID=__lowerCAmelCase						)						)

						for i, example in enumerate(examples[:5]						):
												logger.info("""*** Example ***"""						)
												logger.info(F'''guid: {example}'''						)
												logger.info(F'''features: {features[i]}'''						)

						return features


UpperCamelCase    :      List[Any]  			=		{
    'hans': 3,
}





UpperCamelCase    :      Tuple  			=		{
    'hans': HansProcessor,
}









'''simple docstring'''







import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase    :      List[Any]  			=		get_tests_dir('fixtures/test_sentencepiece_with_bytefallback.model')




@require_sentencepiece
@require_tokenizers
class 		UpperCamelCase__    (a   ,unittest.TestCase			):




		'''simple docstring'''





		_UpperCamelCase				  =					GPTSwaTokenizer
		_UpperCamelCase				  =					False
		_UpperCamelCase				  =					True
		_UpperCamelCase				  =					False






		def 			UpperCamelCase_					(      self       ):
								super().setUp()

								# We have a SentencePiece fixture for testing
								lowerCamelCase__         =  GPTSwaTokenizer(_lowerCAmelCase   ,eos_token="""<unk>"""   ,bos_token="""<unk>"""   ,pad_token="""<unk>"""       )

								tokenizer.save_pretrained(self.tmpdirname       )






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase       ):
								lowerCamelCase__         =  """This is a test"""
								lowerCamelCase__         =  """This is a test"""
								return input_text, output_text






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  """<s>"""
								lowerCamelCase__         =  1

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(_lowerCAmelCase       )   ,_lowerCAmelCase       )
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(_lowerCAmelCase       )   ,_lowerCAmelCase       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  list(self.get_tokenizer().get_vocab().keys()       )

								self.assertEqual(vocab_keys[0]   ,"""<unk>"""       )
								self.assertEqual(vocab_keys[1]   ,"""<s>"""       )
								self.assertEqual(vocab_keys[-1]   ,"""j"""       )
								self.assertEqual(len(_lowerCAmelCase       )   ,20_00       )






		def 			UpperCamelCase_					(      self       ):
								self.assertEqual(self.get_tokenizer().vocab_size   ,20_00       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  GPTSwaTokenizer(_lowerCAmelCase       )

								lowerCamelCase__         =  tokenizer.tokenize("""This is a test"""       )
								self.assertListEqual(_lowerCAmelCase   ,["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]       )

								self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase       )   ,[4_65, 2_87, 2_65, 6_31, 8_42]       )

								lowerCamelCase__         =  tokenizer.tokenize("""I was born in 92000, and this is falsé."""       )
								# fmt: off
								self.assertListEqual(
								    _lowerCAmelCase   ,["""▁I""", """▁was""", """▁bor""", """n""", """▁in""", """▁""", """<0x39>""", """2""", """0""", """0""", """0""", """,""", """▁and""", """▁this""", """▁is""", """▁f""", """al""", """s""", """<0xC3>""", """<0xA9>""", """."""]   ,)
								# fmt: on

								lowerCamelCase__         =  tokenizer.convert_tokens_to_ids(_lowerCAmelCase       )
								self.assertListEqual(
								    _lowerCAmelCase   ,[2_62, 2_72, 15_25, 2_86, 2_71, 2_68, 60, 9_16, 6_33, 6_33, 6_33, 2_59, 2_66, 3_01, 2_87, 3_84, 3_67, 2_63, 1_98, 1_72, 2_60]   ,)

								lowerCamelCase__         =  tokenizer.convert_ids_to_tokens(_lowerCAmelCase       )
								# fmt: off
								self.assertListEqual(
								    _lowerCAmelCase   ,["""▁I""", """▁was""", """▁bor""", """n""", """▁in""", """▁""", """<0x39>""", """2""", """0""", """0""", """0""", """,""", """▁and""", """▁this""", """▁is""", """▁f""", """al""", """s""", """<0xC3>""", """<0xA9>""", """."""]       )
								# fmt: on






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  GPTSwaTokenizer(_lowerCAmelCase       )
								lowerCamelCase__         =  ["""This is a test""", """I was born in 92000, and this is falsé."""]
								lowerCamelCase__         =  [
								    [4_65, 2_87, 2_65, 6_31, 8_42],
								    [2_62, 2_72, 15_25, 2_86, 2_71, 2_68, 60, 9_16, 6_33, 6_33, 6_33, 2_59, 2_66, 3_01, 2_87, 3_84, 3_67, 2_63, 1_98, 1_72, 2_60],
								]

								# Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
								for text, expected_ids in zip(_lowerCAmelCase   ,_lowerCAmelCase       ):
														self.assertListEqual(tokenizer.encode_fast(_lowerCAmelCase       )   ,_lowerCAmelCase       )

								# Test that decode_fast returns the input text
								for text, token_ids in zip(_lowerCAmelCase   ,_lowerCAmelCase       ):
														self.assertEqual(tokenizer.decode_fast(_lowerCAmelCase       )   ,_lowerCAmelCase       )






		@slow
		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  [
								    """<|python|>def fibonacci(n)\n    if n < 0:\n        print('Incorrect input')""",
								    """Hey there, how are you doing this fine day?""",
								    """This is a text with a trailing spaces followed by a dot     .""",
								    """Häj sväjs lillebrör! =)""",
								    """Det är inget fel på Mr. Cool""",
								]

								# fmt: off
								lowerCamelCase__         =  {"""input_ids""": [[6_34_23, 5, 68_11, 1_49_54, 2_82, 8_16, 38_21, 6_34_66, 6_34_25, 6_34_62, 18, 6_39_78, 6_78, 3_01, 13_20, 6_34_23, 6_34_55, 6_34_58, 18, 6_39_82, 42_46, 39_40, 19_01, 4_77_89, 55_47, 1_89_94], [1_96_30, 11_00, 6_34_46, 13_42, 6_33, 5_44, 44_88, 5_93, 51_02, 24_16, 6_34_95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [16_52, 4_28, 2_68, 19_36, 5_15, 2_68, 5_85_93, 2_24_13, 91_06, 5_46, 2_68, 3_32_13, 6_39_79, 6_98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_51_30, 6_34_50, 9_24, 6_34_49, 22_49, 40_62, 15_58, 3_18, 6_35_04, 2_14_98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_09, 3_77, 28_27, 25_59, 3_32, 65_75, 6_34_43, 2_68_01, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
								# fmt: on
								self.tokenizer_integration_test_util(
								    expected_encoding=_lowerCAmelCase   ,model_name="""AI-Sweden/gpt-sw3-126m"""   ,sequences=_lowerCAmelCase   ,)

