



import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


UpperCAmelCase_						=logging.getLogger(__name__)


@dataclass(frozen=__lowerCAmelCase  )
class 							__UpperCamelCase			:





				'''simple docstring'''



				__a							: str
				__a							: str
				__a							: Optional[str]										=None
				__a							: Optional[str]										=None
				__a							: Optional[str]										=None




@dataclass(frozen=__lowerCAmelCase  )
class 							__UpperCamelCase			:





				'''simple docstring'''



				__a							: List[int]
				__a							: Optional[List[int]]										=None
				__a							: Optional[List[int]]										=None
				__a							: Optional[Union[int, float]]										=None
				__a							: Optional[int]										=None


if is_torch_available():
		import torch
		from torch.utils.data import Dataset


		class 							__UpperCamelCase			( __lowerCAmelCase  ):





						'''simple docstring'''



						__a							: List[InputFeatures]



						def __init__( self       , UpperCAmelCase_       , UpperCAmelCase_       , UpperCAmelCase_       , UpperCAmelCase_ = None       , UpperCAmelCase_=False       , UpperCAmelCase_ = False       , ):
													lowerCAmelCase   =		hans_processors[task]()

													lowerCAmelCase   =		os.path.join(
													    lowerCAmelCase_       , '''cached_{}_{}_{}_{}'''.format(
													        '''dev''' if evaluate else '''train'''       , tokenizer.__class__.__name__       , str(lowerCAmelCase_						)       , lowerCAmelCase_       , )       , )
													lowerCAmelCase   =		processor.get_labels()
													if tokenizer.__class__ in (
													    RobertaTokenizer,
													    RobertaTokenizerFast,
													    XLMRobertaTokenizer,
													    BartTokenizer,
													    BartTokenizerFast,
													):
																				# HACK(label indices are swapped in RoBERTa pretrained model)
																				lowerCAmelCase   ,     lowerCAmelCase   =		label_list[2], label_list[1]
													lowerCAmelCase   =		label_list

													# Make sure only the first process in distributed training processes the dataset,
													# and the others will use the cache.
													lowerCAmelCase   =		cached_features_file + '''.lock'''
													with FileLock(lowerCAmelCase_						):
																				if os.path.exists(lowerCAmelCase_						) and not overwrite_cache:
																											logger.info(F"""Loading features from cached file {cached_features_file}"""						)
																											lowerCAmelCase   =		torch.load(lowerCAmelCase_						)
																				else:
																											logger.info(F"""Creating features from dataset file at {data_dir}"""						)

																											lowerCAmelCase   =		(
																											    processor.get_dev_examples(lowerCAmelCase_						) if evaluate else processor.get_train_examples(lowerCAmelCase_						)
																											)

																											logger.info('''Training examples: %s'''       , len(lowerCAmelCase_						)						)
																											lowerCAmelCase   =		hans_convert_examples_to_features(lowerCAmelCase_       , lowerCAmelCase_       , lowerCAmelCase_       , lowerCAmelCase_						)
																											logger.info('''Saving features into cached file %s'''       , lowerCAmelCase_						)
																											torch.save(self.features       , lowerCAmelCase_						)



						def __len__( self						):
													return len(self.features						)



						def __getitem__( self       , UpperCAmelCase_						):
													return self.features[i]






						def       __snake_case		( self						):
													return self.label_list


if is_tf_available():
		import tensorflow as tf


		class 							__UpperCamelCase			:





						'''simple docstring'''



						__a							: List[InputFeatures]



						def __init__( self       , UpperCAmelCase_       , UpperCAmelCase_       , UpperCAmelCase_       , UpperCAmelCase_ = 1_28       , UpperCAmelCase_=False       , UpperCAmelCase_ = False       , ):
													lowerCAmelCase   =		hans_processors[task]()
													lowerCAmelCase   =		processor.get_labels()
													if tokenizer.__class__ in (
													    RobertaTokenizer,
													    RobertaTokenizerFast,
													    XLMRobertaTokenizer,
													    BartTokenizer,
													    BartTokenizerFast,
													):
																				# HACK(label indices are swapped in RoBERTa pretrained model)
																				lowerCAmelCase   ,     lowerCAmelCase   =		label_list[2], label_list[1]
													lowerCAmelCase   =		label_list

													lowerCAmelCase   =		processor.get_dev_examples(lowerCAmelCase_						) if evaluate else processor.get_train_examples(lowerCAmelCase_						)
													lowerCAmelCase   =		hans_convert_examples_to_features(lowerCAmelCase_       , lowerCAmelCase_       , lowerCAmelCase_       , lowerCAmelCase_						)

													def gen():
																				for ex_index, ex in tqdm.tqdm(enumerate(self.features						)       , desc='''convert examples to features'''						):
																											if ex_index % 1_00_00 == 0:
																																		logger.info('''Writing example %d of %d''' % (ex_index, len(lowerCAmelCase_						))						)

																											yield (
																											    {
																											        "example_id": 0,
																											        "input_ids": ex.input_ids,
																											        "attention_mask": ex.attention_mask,
																											        "token_type_ids": ex.token_type_ids,
																											    },
																											    ex.label,
																											)

													lowerCAmelCase   =		tf.data.Dataset.from_generator(
													    lowerCAmelCase_       , (
													        {
													            '''example_id''': tf.intaa,
													            '''input_ids''': tf.intaa,
													            '''attention_mask''': tf.intaa,
													            '''token_type_ids''': tf.intaa,
													        },
													        tf.intaa,
													    )       , (
													        {
													            '''example_id''': tf.TensorShape([]						),
													            '''input_ids''': tf.TensorShape([None, None]						),
													            '''attention_mask''': tf.TensorShape([None, None]						),
													            '''token_type_ids''': tf.TensorShape([None, None]						),
													        },
													        tf.TensorShape([]						),
													    )       , )



						def       __snake_case		( self						):
													return self.dataset



						def __len__( self						):
													return len(self.features						)



						def __getitem__( self       , UpperCAmelCase_						):
													return self.features[i]






						def       __snake_case		( self						):
													return self.label_list


class 							__UpperCamelCase			( __lowerCAmelCase  ):





				'''simple docstring'''



				def       __snake_case		( self       , UpperCAmelCase_						):
											return self._create_examples(self._read_tsv(os.path.join(lowerCAmelCase_       , '''heuristics_train_set.txt'''						)						)       , '''train'''						)



				def       __snake_case		( self       , UpperCAmelCase_						):
											return self._create_examples(self._read_tsv(os.path.join(lowerCAmelCase_       , '''heuristics_evaluation_set.txt'''						)						)       , '''dev'''						)



				def       __snake_case		( self						):
											return ["contradiction", "entailment", "neutral"]






				def       __snake_case		( self       , UpperCAmelCase_       , UpperCAmelCase_						):
											lowerCAmelCase   =		[]
											for i, line in enumerate(lowerCAmelCase_						):
																		if i == 0:
																									continue
																		lowerCAmelCase   =		'''%s-%s''' % (set_type, line[0])
																		lowerCAmelCase   =		line[5]
																		lowerCAmelCase   =		line[6]
																		lowerCAmelCase   =		line[7][2:] if line[7].startswith('''ex'''						) else line[7]
																		lowerCAmelCase   =		line[0]
																		examples.append(InputExample(guid=lowerCAmelCase_       , text_a=lowerCAmelCase_       , text_b=lowerCAmelCase_       , label=lowerCAmelCase_       , pairID=lowerCAmelCase_						)						)
											return examples




def       UpperCAmelCase       (     _snake_case						, _snake_case						, _snake_case						, _snake_case						, ):
							lowerCAmelCase   =		{label: i for i, label in enumerate(snake_case__							)}

							lowerCAmelCase   =		[]
							for ex_index, example in tqdm.tqdm(enumerate(snake_case__							)						, desc='''convert examples to features'''							):
														if ex_index % 10000 == 0:
																					logger.info('''Writing example %d''' % (ex_index)							)

														lowerCAmelCase   =		tokenizer(
														    example.text_a						, example.text_b						, add_special_tokens=snake_case__						, max_length=snake_case__						, padding='''max_length'''						, truncation=snake_case__						, return_overflowing_tokens=snake_case__						, )

														lowerCAmelCase   =		label_map[example.label] if example.label in label_map else 0

														lowerCAmelCase   =		int(example.pairID							)

														features.append(InputFeatures(**snake_case__						, label=snake_case__						, pairID=snake_case__							)							)

							for i, example in enumerate(examples[:5]							):
														logger.info('''*** Example ***'''							)
														logger.info(F"""guid: {example}"""							)
														logger.info(F"""features: {features[i]}"""							)

							return features


UpperCAmelCase_						={
    """hans""": 3,
}

UpperCAmelCase_						={
    """hans""": HansProcessor,
}









from typing import Optional

import pyspark

from .. import Features, NamedSplit
from ..download import DownloadMode
from ..packaged_modules.spark.spark import Spark
from .abc import AbstractDatasetReader


class 							__UpperCamelCase			( __UpperCAmelCase  ):





				'''simple docstring'''



				def __init__( self       , UpperCAmelCase_       , UpperCAmelCase_ = None       , UpperCAmelCase_ = None       , UpperCAmelCase_ = True       , UpperCAmelCase_ = None       , UpperCAmelCase_ = False       , UpperCAmelCase_ = None       , UpperCAmelCase_ = True       , UpperCAmelCase_ = "arrow"       , **UpperCAmelCase_       , ):
											super().__init__(
											    split=UpperCAmelCase_       , features=UpperCAmelCase_       , cache_dir=UpperCAmelCase_       , keep_in_memory=UpperCAmelCase_       , streaming=UpperCAmelCase_       , **UpperCAmelCase_       , )
											lowerCAmelCase   =		load_from_cache_file
											lowerCAmelCase   =		file_format
											lowerCAmelCase   =		Spark(
											    df=UpperCAmelCase_       , features=UpperCAmelCase_       , cache_dir=UpperCAmelCase_       , working_dir=UpperCAmelCase_       , **UpperCAmelCase_       , )






				def       __snake_case		( self						):
											if self.streaming:
																		return self.builder.as_streaming_dataset(split=self.split						)
											lowerCAmelCase   =		None if self._load_from_cache_file else DownloadMode.FORCE_REDOWNLOAD
											self.builder.download_and_prepare(
											    download_mode=UpperCAmelCase_       , file_format=self._file_format       , )
											return self.builder.as_dataset(split=self.split						)



