


"""simple docstring"""
import logging
import os
from dataclasses import dataclass, field
from typing import Dict, Optional

import datasets
import numpy as np
import tensorflow as tf

from transformers import (
    AutoConfig,
    AutoTokenizer,
    EvalPrediction,
    HfArgumentParser,
    PreTrainedTokenizer,
    TFAutoModelForSequenceClassification,
    TFTrainer,
    TFTrainingArguments,
)
from transformers.utils import logging as hf_logging


hf_logging.set_verbosity_info()
hf_logging.enable_default_handler()
hf_logging.enable_explicit_format()







def 						_a				(      _SCREAMING_SNAKE_CASE				, _SCREAMING_SNAKE_CASE				, _SCREAMING_SNAKE_CASE				, _SCREAMING_SNAKE_CASE				, _SCREAMING_SNAKE_CASE				, _SCREAMING_SNAKE_CASE = None				, )    ->				Union[str, Any]:
							snake_case_           =		{}

							if train_file is not None:
														snake_case_           =		[train_file]
							if eval_file is not None:
														snake_case_           =		[eval_file]
							if test_file is not None:
														snake_case_           =		[test_file]

							snake_case_           =		datasets.load_dataset("""csv"""				, data_files=_SCREAMING_SNAKE_CASE				)
							snake_case_           =		list(ds[list(files.keys()				)[0]].features.keys()				)
							snake_case_           =		features_name.pop(_SCREAMING_SNAKE_CASE				)
							snake_case_           =		list(set(ds[list(files.keys()				)[0]][label_name]				)				)
							snake_case_           =		{label: i for i, label in enumerate(_SCREAMING_SNAKE_CASE				)}
							snake_case_           =		tokenizer.model_input_names
							snake_case_           =		{}

							if len(_SCREAMING_SNAKE_CASE				) == 1:
														for k in files.keys():
																					snake_case_           =		ds[k].map(
																					    lambda _SCREAMING_SNAKE_CASE				: tokenizer.batch_encode_plus(
																					        example[features_name[0]]				, truncation=_SCREAMING_SNAKE_CASE				, max_length=_SCREAMING_SNAKE_CASE				, padding="""max_length"""				)				, batched=_SCREAMING_SNAKE_CASE				, )
							elif len(_SCREAMING_SNAKE_CASE				) == 2:
														for k in files.keys():
																					snake_case_           =		ds[k].map(
																					    lambda _SCREAMING_SNAKE_CASE				: tokenizer.batch_encode_plus(
																					        (example[features_name[0]], example[features_name[1]])				, truncation=_SCREAMING_SNAKE_CASE				, max_length=_SCREAMING_SNAKE_CASE				, padding="""max_length"""				, )				, batched=_SCREAMING_SNAKE_CASE				, )

							def gen_train():
														for ex in transformed_ds[datasets.Split.TRAIN]:
																					snake_case_           =		{k: v for k, v in ex.items() if k in input_names}
																					snake_case_           =		labelaid[ex[label_name]]
																					yield (d, label)

							def gen_val():
														for ex in transformed_ds[datasets.Split.VALIDATION]:
																					snake_case_           =		{k: v for k, v in ex.items() if k in input_names}
																					snake_case_           =		labelaid[ex[label_name]]
																					yield (d, label)

							def gen_test():
														for ex in transformed_ds[datasets.Split.TEST]:
																					snake_case_           =		{k: v for k, v in ex.items() if k in input_names}
																					snake_case_           =		labelaid[ex[label_name]]
																					yield (d, label)

							snake_case_           =		(
							    tf.data.Dataset.from_generator(
							        _SCREAMING_SNAKE_CASE				, ({k: tf.intaa for k in input_names}, tf.intaa)				, ({k: tf.TensorShape([None]				) for k in input_names}, tf.TensorShape([]				))				, )
							    if datasets.Split.TRAIN in transformed_ds
							    else None
							)

							if train_ds is not None:
														snake_case_           =		train_ds.apply(tf.data.experimental.assert_cardinality(len(ds[datasets.Split.TRAIN]				)				)				)

							snake_case_           =		(
							    tf.data.Dataset.from_generator(
							        _SCREAMING_SNAKE_CASE				, ({k: tf.intaa for k in input_names}, tf.intaa)				, ({k: tf.TensorShape([None]				) for k in input_names}, tf.TensorShape([]				))				, )
							    if datasets.Split.VALIDATION in transformed_ds
							    else None
							)

							if val_ds is not None:
														snake_case_           =		val_ds.apply(tf.data.experimental.assert_cardinality(len(ds[datasets.Split.VALIDATION]				)				)				)

							snake_case_           =		(
							    tf.data.Dataset.from_generator(
							        _SCREAMING_SNAKE_CASE				, ({k: tf.intaa for k in input_names}, tf.intaa)				, ({k: tf.TensorShape([None]				) for k in input_names}, tf.TensorShape([]				))				, )
							    if datasets.Split.TEST in transformed_ds
							    else None
							)

							if test_ds is not None:
														snake_case_           =		test_ds.apply(tf.data.experimental.assert_cardinality(len(ds[datasets.Split.TEST]				)				)				)

							return train_ds, val_ds, test_ds, labelaid


__SCREAMING_SNAKE_CASE							:    Optional[Any]		=    logging.getLogger(__name__)





@dataclass
class      __A     :





							'''simple docstring'''

							__lowercase:		int							      =    field(metadata={"""help""": """Which column contains the label"""})
							__lowercase:		str							      =    field(default=snake_case__						,		metadata={"""help""": """The path of the training file"""})
							__lowercase:		Optional[str]							      =    field(default=snake_case__						,		metadata={"""help""": """The path of the development file"""})
							__lowercase:		Optional[str]							      =    field(default=snake_case__						,		metadata={"""help""": """The path of the test file"""})
							__lowercase:		int							      =    field(
							    default=1_28						,		metadata={
							        """help""": (
							            """The maximum total input sequence length after tokenization. Sequences longer """
							            """than this will be truncated, sequences shorter will be padded."""
							        )
							    }						,		)
							__lowercase:		bool							      =    field(
							    default=snake_case__						,		metadata={"""help""": """Overwrite the cached training and evaluation sets"""})





@dataclass
class      __A     :





							'''simple docstring'''

							__lowercase:		str							      =    field(
							    metadata={"""help""": """Path to pretrained model or model identifier from huggingface.co/models"""})
							__lowercase:		Optional[str]							      =    field(
							    default=snake_case__						,		metadata={"""help""": """Pretrained config name or path if not the same as model_name"""})
							__lowercase:		Optional[str]							      =    field(
							    default=snake_case__						,		metadata={"""help""": """Pretrained tokenizer name or path if not the same as model_name"""})
							__lowercase:		bool							      =    field(default=snake_case__						,		metadata={"""help""": """Set this flag to use fast tokenization."""})
							# If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
							# or just modify its tokenizer_config.json.
							__lowercase:		Optional[str]							      =    field(
							    default=snake_case__						,		metadata={"""help""": """Where do you want to store the pretrained models downloaded from huggingface.co"""}						,		)







def 						_a				(      )    ->				Union[str, Any]:
							# See all possible arguments in src/transformers/training_args.py
							# or by passing the --help flag to this script.
							# We now keep distinct sets of args, for a cleaner separation of concerns.
							snake_case_           =		HfArgumentParser((ModelArguments, DataTrainingArguments, TFTrainingArguments)				)
							snake_case_       ,						snake_case_       ,						snake_case_           =		parser.parse_args_into_dataclasses()

							if (
							    os.path.exists(training_args.output_dir				)
							    and os.listdir(training_args.output_dir				)
							    and training_args.do_train
							    and not training_args.overwrite_output_dir
							):
														raise ValueError(
														    f"""Output directory ({training_args.output_dir}) already exists and is not empty. Use"""
														    """ --overwrite_output_dir to overcome."""				)

							# Setup logging
							logging.basicConfig(
							    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s"""				, datefmt="""%m/%d/%Y %H:%M:%S"""				, level=logging.INFO				, )
							logger.info(
							    f"""n_replicas: {training_args.n_replicas}, distributed training: {bool(training_args.n_replicas > 1				)}, """
							    f"""16-bits training: {training_args.fpaa}"""				)
							logger.info(f"""Training/evaluation parameters {training_args}"""				)

							# Load pretrained model and tokenizer
							#
							# Distributed training:
							# The .from_pretrained methods guarantee that only one local process can concurrently
							# download model & vocab.

							snake_case_           =		AutoTokenizer.from_pretrained(
							    model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path				, cache_dir=model_args.cache_dir				, )

							snake_case_       ,						snake_case_       ,						snake_case_       ,						snake_case_           =		get_tfds(
							    train_file=data_args.train_file				, eval_file=data_args.dev_file				, test_file=data_args.test_file				, tokenizer=_SCREAMING_SNAKE_CASE				, label_column_id=data_args.label_column_id				, max_seq_length=data_args.max_seq_length				, )

							snake_case_           =		AutoConfig.from_pretrained(
							    model_args.config_name if model_args.config_name else model_args.model_name_or_path				, num_labels=len(_SCREAMING_SNAKE_CASE				)				, labelaid=_SCREAMING_SNAKE_CASE				, idalabel={id: label for label, id in labelaid.items()}				, finetuning_task="""text-classification"""				, cache_dir=model_args.cache_dir				, )

							with training_args.strategy.scope():
														snake_case_           =		TFAutoModelForSequenceClassification.from_pretrained(
														    model_args.model_name_or_path				, from_pt=bool(""".bin""" in model_args.model_name_or_path				)				, config=_SCREAMING_SNAKE_CASE				, cache_dir=model_args.cache_dir				, )

							def compute_metrics(_SCREAMING_SNAKE_CASE				) -> Dict:
														snake_case_           =		np.argmax(p.predictions				, axis=1				)

														return {"acc": (preds == p.label_ids).mean()}

							# Initialize our Trainer
							snake_case_           =		TFTrainer(
							    model=_SCREAMING_SNAKE_CASE				, args=_SCREAMING_SNAKE_CASE				, train_dataset=_SCREAMING_SNAKE_CASE				, eval_dataset=_SCREAMING_SNAKE_CASE				, compute_metrics=_SCREAMING_SNAKE_CASE				, )

							# Training
							if training_args.do_train:
														trainer.train()
														trainer.save_model()
														tokenizer.save_pretrained(training_args.output_dir				)

							# Evaluation
							snake_case_           =		{}
							if training_args.do_eval:
														logger.info("""*** Evaluate ***"""				)
														snake_case_           =		trainer.evaluate()
														snake_case_           =		os.path.join(training_args.output_dir				, """eval_results.txt"""				)

														with open(_SCREAMING_SNAKE_CASE				, """w"""				) as writer:
																					logger.info("""***** Eval results *****"""				)

																					for key, value in result.items():
																												logger.info(f"""  {key} = {value}"""				)
																												writer.write(f"""{key} = {value}\n"""				)

																					results.update(_SCREAMING_SNAKE_CASE				)

							return results


if __name__ == "__main__":
				main()







"""simple docstring"""
import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__SCREAMING_SNAKE_CASE							:    List[str]		=    logging.get_logger(__name__)
__SCREAMING_SNAKE_CASE							:    Optional[int]		=    {'vocab_file': 'spiece.model'}

__SCREAMING_SNAKE_CASE							:    List[str]		=    {
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    }
}

__SCREAMING_SNAKE_CASE							:    List[str]		=    {
    'albert-base-v1': 512,
    'albert-large-v1': 512,
    'albert-xlarge-v1': 512,
    'albert-xxlarge-v1': 512,
    'albert-base-v2': 512,
    'albert-large-v2': 512,
    'albert-xlarge-v2': 512,
    'albert-xxlarge-v2': 512,
}

__SCREAMING_SNAKE_CASE							:    int		=    '▁'





class      __A     (snake_case__):





							'''simple docstring'''

							__lowercase:		Optional[Any]							      =    VOCAB_FILES_NAMES
							__lowercase:		Optional[int]							      =    PRETRAINED_VOCAB_FILES_MAP
							__lowercase:		Tuple							      =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



							def __init__(				self							:      int		,       UpperCAmelCase_							:      Any		,       UpperCAmelCase_							:      Dict=True		,       UpperCAmelCase_							:      Dict=True		,       UpperCAmelCase_							:      List[str]=False		,       UpperCAmelCase_							:      List[Any]="[CLS]"		,       UpperCAmelCase_							:      Any="[SEP]"		,       UpperCAmelCase_							:      str="<unk>"		,       UpperCAmelCase_							:      str="[SEP]"		,       UpperCAmelCase_							:      Optional[Any]="<pad>"		,       UpperCAmelCase_							:      Optional[int]="[CLS]"		,       UpperCAmelCase_							:      int="[MASK]"		,       UpperCAmelCase_							:      Optional[Dict[str, Any]] = None		,       **UpperCAmelCase_							:      Union[str, Any]		,       )					->None:



														"""simple docstring"""



														snake_case_           =		(
														    AddedToken(UpperCAmelCase_		,       lstrip=UpperCAmelCase_		,       rstrip=UpperCAmelCase_		,       normalized=UpperCAmelCase_       )
														    if isinstance(UpperCAmelCase_		,       UpperCAmelCase_       )
														    else mask_token
														)

														snake_case_           =		{} if sp_model_kwargs is None else sp_model_kwargs

														super().__init__(
														    do_lower_case=UpperCAmelCase_		,       remove_space=UpperCAmelCase_		,       keep_accents=UpperCAmelCase_		,       bos_token=UpperCAmelCase_		,       eos_token=UpperCAmelCase_		,       unk_token=UpperCAmelCase_		,       sep_token=UpperCAmelCase_		,       pad_token=UpperCAmelCase_		,       cls_token=UpperCAmelCase_		,       mask_token=UpperCAmelCase_		,       sp_model_kwargs=self.sp_model_kwargs		,       **UpperCAmelCase_		,       )

														snake_case_           =		do_lower_case
														snake_case_           =		remove_space
														snake_case_           =		keep_accents
														snake_case_           =		vocab_file

														snake_case_           =		spm.SentencePieceProcessor(**self.sp_model_kwargs       )
														self.sp_model.Load(UpperCAmelCase_       )



							@property
							def 		lowerCAmelCase       (				self							:      List[Any]       )					->Dict:



														"""simple docstring"""



														return len(self.sp_model       )



							def 		lowerCAmelCase       (				self							:      str       )					->List[Any]:



														"""simple docstring"""



														snake_case_           =		{self.convert_ids_to_tokens(UpperCAmelCase_       ): i for i in range(self.vocab_size       )}
														vocab.update(self.added_tokens_encoder       )
														return vocab



							def __getstate__(				self							:      Dict       )					->List[str]:



														"""simple docstring"""



														snake_case_           =		self.__dict__.copy()
														snake_case_           =		None
														return state



							def __setstate__(				self							:      Tuple		,       UpperCAmelCase_							:      Optional[int]       )					->Optional[int]:



														"""simple docstring"""



														snake_case_           =		d

														# for backward compatibility
														if not hasattr(self		,       """sp_model_kwargs"""       ):
																					snake_case_           =		{}

														snake_case_           =		spm.SentencePieceProcessor(**self.sp_model_kwargs       )
														self.sp_model.Load(self.vocab_file       )



							def 		lowerCAmelCase       (				self							:      Tuple		,       UpperCAmelCase_							:      Any       )					->str:



														"""simple docstring"""



														if self.remove_space:
																					snake_case_           =		""" """.join(inputs.strip().split()       )
														else:
																					snake_case_           =		inputs
														snake_case_           =		outputs.replace("""``"""		,       """\""""       ).replace("""''"""		,       """\""""       )

														if not self.keep_accents:
																					snake_case_           =		unicodedata.normalize("""NFKD"""		,       UpperCAmelCase_       )
																					snake_case_           =		"""""".join([c for c in outputs if not unicodedata.combining(UpperCAmelCase_       )]       )
														if self.do_lower_case:
																					snake_case_           =		outputs.lower()

														return outputs



							def 		lowerCAmelCase       (				self							:      Dict		,       UpperCAmelCase_							:      str       )					->List[str]:



														"""simple docstring"""



														snake_case_           =		self.preprocess_text(UpperCAmelCase_       )
														snake_case_           =		self.sp_model.encode(UpperCAmelCase_		,       out_type=UpperCAmelCase_       )
														snake_case_           =		[]
														for piece in pieces:
																					if len(UpperCAmelCase_       ) > 1 and piece[-1] == str(""","""       ) and piece[-2].isdigit():
																												snake_case_           =		self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCAmelCase_		,       """"""       )       )
																												if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																																			if len(cur_pieces[0]       ) == 1:
																																										snake_case_           =		cur_pieces[1:]
																																			else:
																																										snake_case_           =		cur_pieces[0][1:]
																												cur_pieces.append(piece[-1]       )
																												new_pieces.extend(UpperCAmelCase_       )
																					else:
																												new_pieces.append(UpperCAmelCase_       )

														return new_pieces



							def 		lowerCAmelCase       (				self							:      Optional[int]		,       UpperCAmelCase_							:      Optional[int]       )					->Dict:



														"""simple docstring"""



														return self.sp_model.PieceToId(UpperCAmelCase_       )



							def 		lowerCAmelCase       (				self							:      str		,       UpperCAmelCase_							:      Union[str, Any]       )					->Optional[int]:



														"""simple docstring"""



														return self.sp_model.IdToPiece(UpperCAmelCase_       )



							def 		lowerCAmelCase       (				self							:      str		,       UpperCAmelCase_							:      Dict       )					->Any:



														"""simple docstring"""



														snake_case_           =		[]
														snake_case_           =		""""""
														snake_case_           =		False
														for token in tokens:
																					# make sure that special tokens are not decoded using sentencepiece model
																					if token in self.all_special_tokens:
																												if not prev_is_special:
																																			out_string += " "
																												out_string += self.sp_model.decode(UpperCAmelCase_       ) + token
																												snake_case_           =		True
																												snake_case_           =		[]
																					else:
																												current_sub_tokens.append(UpperCAmelCase_       )
																												snake_case_           =		False
														out_string += self.sp_model.decode(UpperCAmelCase_       )
														return out_string.strip()



							def 		lowerCAmelCase       (				self							:      Tuple		,       UpperCAmelCase_							:      List[int]		,       UpperCAmelCase_							:      Optional[List[int]] = None       )					->List[int]:



														"""simple docstring"""



														snake_case_           =		[self.sep_token_id]
														snake_case_           =		[self.cls_token_id]
														if token_ids_a is None:
																					return cls + token_ids_a + sep
														return cls + token_ids_a + sep + token_ids_a + sep



							def 		lowerCAmelCase       (				self							:      Tuple		,       UpperCAmelCase_							:      List[int]		,       UpperCAmelCase_							:      Optional[List[int]] = None		,       UpperCAmelCase_							:      bool = False       )					->List[int]:



														"""simple docstring"""



														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=UpperCAmelCase_		,       token_ids_a=UpperCAmelCase_		,       already_has_special_tokens=UpperCAmelCase_       )

														if token_ids_a is not None:
																					return [1] + ([0] * len(UpperCAmelCase_       )) + [1] + ([0] * len(UpperCAmelCase_       )) + [1]
														return [1] + ([0] * len(UpperCAmelCase_       )) + [1]



							def 		lowerCAmelCase       (				self							:      Dict		,       UpperCAmelCase_							:      List[int]		,       UpperCAmelCase_							:      Optional[List[int]] = None       )					->List[int]:



														"""simple docstring"""



														snake_case_           =		[self.sep_token_id]
														snake_case_           =		[self.cls_token_id]

														if token_ids_a is None:
																					return len(cls + token_ids_a + sep       ) * [0]
														return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]



							def 		lowerCAmelCase       (				self							:      str		,       UpperCAmelCase_							:      str		,       UpperCAmelCase_							:      Optional[str] = None       )					->Tuple[str]:



														"""simple docstring"""



														if not os.path.isdir(UpperCAmelCase_       ):
																					logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""       )
																					return
														snake_case_           =		os.path.join(
														    UpperCAmelCase_		,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]       )

														if os.path.abspath(self.vocab_file       ) != os.path.abspath(UpperCAmelCase_       ) and os.path.isfile(self.vocab_file       ):
																					copyfile(self.vocab_file		,       UpperCAmelCase_       )
														elif not os.path.isfile(self.vocab_file       ):
																					with open(UpperCAmelCase_		,       """wb"""       ) as fi:
																												snake_case_           =		self.sp_model.serialized_model_proto()
																												fi.write(UpperCAmelCase_       )

														return (out_vocab_file,)



