
"""simple docstring"""



import os
from argparse import ArgumentParser, Namespace

from ..data import SingleSentenceClassificationProcessor as Processor
from ..pipelines import TextClassificationPipeline
from ..utils import is_tf_available, is_torch_available, logging
from . import BaseTransformersCLICommand


if not is_tf_available() and not is_torch_available():
						raise RuntimeError('''At least one of PyTorch or TensorFlow 2.0+ should be installed to use CLI training''')

# TF training parameters
_snake_case          =						False
_snake_case          =						False






def 	__snake_case  (    SCREAMING_SNAKE_CASE:					Namespace						):






							"""simple docstring"""



							return TrainCommand(SCREAMING_SNAKE_CASE						)

class        _SCREAMING_SNAKE_CASE  (  UpperCAmelCase							):






				'''simple docstring'''



				@staticmethod
				def 				__lowerCamelCase	(					UpperCAmelCase_	:			ArgumentParser  )		->  int:







											"""simple docstring"""




											_lowerCAmelCase  	=			parser.add_parser('train'				,     help='CLI tool to train a model on a task.'  )

											train_parser.add_argument(
											    '--train_data'				,     type=UpperCAmelCase_				,     required=UpperCAmelCase_				,     help='path to train (and optionally evaluation) dataset as a csv with tab separated labels and sentences.'				,     )
											train_parser.add_argument(
											    '--column_label'				,     type=UpperCAmelCase_				,     default=0				,     help='Column of the dataset csv file with example labels.'  )
											train_parser.add_argument(
											    '--column_text'				,     type=UpperCAmelCase_				,     default=1				,     help='Column of the dataset csv file with example texts.'  )
											train_parser.add_argument(
											    '--column_id'				,     type=UpperCAmelCase_				,     default=2				,     help='Column of the dataset csv file with example ids.'  )
											train_parser.add_argument(
											    '--skip_first_row'				,     action='store_true'				,     help='Skip the first row of the csv file (headers).'  )

											train_parser.add_argument('--validation_data'				,     type=UpperCAmelCase_				,     default=''				,     help='path to validation dataset.'  )
											train_parser.add_argument(
											    '--validation_split'				,     type=UpperCAmelCase_				,     default=0.1				,     help='if validation dataset is not provided, fraction of train dataset to use as validation dataset.'				,     )

											train_parser.add_argument('--output'				,     type=UpperCAmelCase_				,     default='./'				,     help='path to saved the trained model.'  )

											train_parser.add_argument(
											    '--task'				,     type=UpperCAmelCase_				,     default='text_classification'				,     help='Task to train the model on.'  )
											train_parser.add_argument(
											    '--model'				,     type=UpperCAmelCase_				,     default='bert-base-uncased'				,     help='Model\'s name or path to stored model.'  )
											train_parser.add_argument('--train_batch_size'				,     type=UpperCAmelCase_				,     default=32				,     help='Batch size for training.'  )
											train_parser.add_argument('--valid_batch_size'				,     type=UpperCAmelCase_				,     default=64				,     help='Batch size for validation.'  )
											train_parser.add_argument('--learning_rate'				,     type=UpperCAmelCase_				,     default=3E-5				,     help='Learning rate.'  )
											train_parser.add_argument('--adam_epsilon'				,     type=UpperCAmelCase_				,     default=1E-0_8				,     help='Epsilon for Adam optimizer.'  )
											train_parser.set_defaults(func=UpperCAmelCase_  )


				def __init__(					self	:			Tuple				,     UpperCAmelCase_	:			Namespace  )		->  Dict:







											"""simple docstring"""




											_lowerCAmelCase  	=			logging.get_logger('transformers-cli/training'  )

											_lowerCAmelCase  	=			'tf' if is_tf_available() else 'torch'

											os.makedirs(args.output				,     exist_ok=UpperCAmelCase_  )
											_lowerCAmelCase  	=			args.output

											_lowerCAmelCase  	=			args.column_label
											_lowerCAmelCase  	=			args.column_text
											_lowerCAmelCase  	=			args.column_id

											self.logger.info(F"""Loading {args.task} pipeline for {args.model}"""  )
											if args.task == "text_classification":
																		_lowerCAmelCase  	=			TextClassificationPipeline.from_pretrained(args.model  )
											elif args.task == "token_classification":
																		raise NotImplementedError
											elif args.task == "question_answering":
																		raise NotImplementedError

											self.logger.info(F"""Loading dataset from {args.train_data}"""  )
											_lowerCAmelCase  	=			Processor.create_from_csv(
											    args.train_data				,     column_label=args.column_label				,     column_text=args.column_text				,     column_id=args.column_id				,     skip_first_row=args.skip_first_row				,     )
											_lowerCAmelCase  	=			None
											if args.validation_data:
																		self.logger.info(F"""Loading validation dataset from {args.validation_data}"""  )
																		_lowerCAmelCase  	=			Processor.create_from_csv(
																		    args.validation_data				,     column_label=args.column_label				,     column_text=args.column_text				,     column_id=args.column_id				,     skip_first_row=args.skip_first_row				,     )

											_lowerCAmelCase  	=			args.validation_split
											_lowerCAmelCase  	=			args.train_batch_size
											_lowerCAmelCase  	=			args.valid_batch_size
											_lowerCAmelCase  	=			args.learning_rate
											_lowerCAmelCase  	=			args.adam_epsilon


				def 				__lowerCamelCase	(					self	:			List[Any]  )		->  str:







											"""simple docstring"""




											if self.framework == "tf":
																		return self.run_tf()
											return self.run_torch()


				def 				__lowerCamelCase	(					self	:			Union[str, Any]  )		->  str:







											"""simple docstring"""




											raise NotImplementedError







				def 				__lowerCamelCase	(					self	:			List[str]  )		->  Optional[Any]:







											"""simple docstring"""




											self.pipeline.fit(
											    self.train_dataset				,     validation_data=self.valid_dataset				,     validation_split=self.validation_split				,     learning_rate=self.learning_rate				,     adam_epsilon=self.adam_epsilon				,     train_batch_size=self.train_batch_size				,     valid_batch_size=self.valid_batch_size				,     )

											# Save trained pipeline
											self.pipeline.save_pretrained(self.output  )








"""simple docstring"""



import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class        _SCREAMING_SNAKE_CASE  (  UpperCAmelCase      ,				unittest.TestCase							):






				'''simple docstring'''



				SCREAMING_SNAKE_CASE_:							str							=					None
				SCREAMING_SNAKE_CASE_:							List[str]							=					BloomTokenizerFast
				SCREAMING_SNAKE_CASE_:							List[str]							=					BloomTokenizerFast
				SCREAMING_SNAKE_CASE_:							Any							=					True
				SCREAMING_SNAKE_CASE_:							List[Any]							=					False
				SCREAMING_SNAKE_CASE_:							Optional[int]							=					"tokenizer_file"
				SCREAMING_SNAKE_CASE_:							str							=					{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}


				def 				__lowerCamelCase	(					self	:			Tuple  )		->  str:







											"""simple docstring"""




											super().setUp()
											_lowerCAmelCase  	=			BloomTokenizerFast.from_pretrained('bigscience/tokenizer'  )
											tokenizer.save_pretrained(self.tmpdirname  )


				def 				__lowerCamelCase	(					self	:			List[str]				,     **UpperCAmelCase_	:			Any  )		->  Optional[Any]:







											"""simple docstring"""




											kwargs.update(self.special_tokens_map  )
											return BloomTokenizerFast.from_pretrained(self.tmpdirname				,     **UpperCAmelCase_  )


				def 				__lowerCamelCase	(					self	:			List[Any]  )		->  str:







											"""simple docstring"""




											_lowerCAmelCase  	=			self.get_rust_tokenizer()

											_lowerCAmelCase  	=			['The quick brown fox</s>', 'jumps over the lazy dog</s>']
											_lowerCAmelCase  	=			[[2_175, 23_714, 73_173, 144_252, 2], [77, 132_619, 3_478, 368, 109_586, 35_433, 2]]

											_lowerCAmelCase  	=			tokenizer.batch_encode_plus(UpperCAmelCase_  )['input_ids']
											self.assertListEqual(UpperCAmelCase_				,     UpperCAmelCase_  )

											_lowerCAmelCase  	=			tokenizer.batch_decode(UpperCAmelCase_  )
											self.assertListEqual(UpperCAmelCase_				,     UpperCAmelCase_  )


				def 				__lowerCamelCase	(					self	:			Any				,     UpperCAmelCase_	:			List[str]=6  )		->  Any:







											"""simple docstring"""




											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""  ):
																									_lowerCAmelCase  	=			self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_				,     **UpperCAmelCase_  )
																									# tokenizer_r.pad_token = None # Hotfixing padding = None
																									# Simple input
																									_lowerCAmelCase  	=			'This is a simple input'
																									_lowerCAmelCase  	=			['This is a simple input 1', 'This is a simple input 2']
																									_lowerCAmelCase  	=			('This is a simple input', 'This is a pair')
																									_lowerCAmelCase  	=			[
																									    ('This is a simple input 1', 'This is a simple input 2'),
																									    ('This is a simple pair 1', 'This is a simple pair 2'),
																									]

																									# Simple input tests
																									try:
																																tokenizer_r.encode(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )
																																tokenizer_r.encode_plus(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )

																																tokenizer_r.batch_encode_plus(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )
																																tokenizer_r.encode(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )
																																tokenizer_r.batch_encode_plus(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )
																									except ValueError:
																																self.fail('Bloom Tokenizer should be able to deal with padding'  )

																									_lowerCAmelCase  	=			None  # Hotfixing padding = None
																									self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'  )

																									# Simple input
																									self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'  )

																									# Simple input
																									self.assertRaises(
																									    UpperCAmelCase_				,     tokenizer_r.batch_encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'				,     )

																									# Pair input
																									self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'  )

																									# Pair input
																									self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'  )

																									# Pair input
																									self.assertRaises(
																									    UpperCAmelCase_				,     tokenizer_r.batch_encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'				,     )


				def 				__lowerCamelCase	(					self	:			str  )		->  Any:







											"""simple docstring"""




											_lowerCAmelCase  	=			self.get_rust_tokenizer()
											_lowerCAmelCase  	=			load_dataset('xnli'				,     'all_languages'				,     split='test'				,     streaming=UpperCAmelCase_  )

											_lowerCAmelCase  	=			next(iter(UpperCAmelCase_  )  )['premise']  # pick up one data
											_lowerCAmelCase  	=			list(sample_data.values()  )

											_lowerCAmelCase  	=			list(map(tokenizer.encode				,     UpperCAmelCase_  )  )
											_lowerCAmelCase  	=			[tokenizer.decode(UpperCAmelCase_				,     clean_up_tokenization_spaces=UpperCAmelCase_  ) for x in output_tokens]
											self.assertListEqual(UpperCAmelCase_				,     UpperCAmelCase_  )







				def 				__lowerCamelCase	(					self	:			Tuple  )		->  List[Any]:







											"""simple docstring"""




											self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map  )				,     1  )
											self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()  )[0]  )				,     1  )






