



import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__lowerCamelCase         =			logging.get_logger(__name__)

__lowerCamelCase         =			{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

__lowerCamelCase         =			{
    '''vocab_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json'''},
    '''merges_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt'''},
}

__lowerCamelCase         =			{
    '''ctrl''': 256,
}

__lowerCamelCase         =			{
    '''Pregnancy''': 168629,
    '''Christianity''': 7675,
    '''Explain''': 106423,
    '''Fitness''': 63440,
    '''Saving''': 63163,
    '''Ask''': 27171,
    '''Ass''': 95985,
    '''Joke''': 163509,
    '''Questions''': 45622,
    '''Thoughts''': 49605,
    '''Retail''': 52342,
    '''Feminism''': 164338,
    '''Writing''': 11992,
    '''Atheism''': 192263,
    '''Netflix''': 48616,
    '''Computing''': 39639,
    '''Opinion''': 43213,
    '''Alone''': 44967,
    '''Funny''': 58917,
    '''Gaming''': 40358,
    '''Human''': 4088,
    '''India''': 1331,
    '''Joker''': 77138,
    '''Diet''': 36206,
    '''Legal''': 11859,
    '''Norman''': 4939,
    '''Tip''': 72689,
    '''Weight''': 52343,
    '''Movies''': 46273,
    '''Running''': 23425,
    '''Science''': 2090,
    '''Horror''': 37793,
    '''Confession''': 60572,
    '''Finance''': 12250,
    '''Politics''': 16360,
    '''Scary''': 191985,
    '''Support''': 12654,
    '''Technologies''': 32516,
    '''Teenage''': 66160,
    '''Event''': 32769,
    '''Learned''': 67460,
    '''Notion''': 182770,
    '''Wikipedia''': 37583,
    '''Books''': 6665,
    '''Extract''': 76050,
    '''Confessions''': 102701,
    '''Conspiracy''': 75932,
    '''Links''': 63674,
    '''Narcissus''': 150425,
    '''Relationship''': 54766,
    '''Relationships''': 134796,
    '''Reviews''': 41671,
    '''News''': 4256,
    '''Translation''': 26820,
    '''multilingual''': 128406,
}







def    _snake_case (       __snake_case    )     ->     str:






      '''simple docstring'''







      UpperCAmelCase_  :	Tuple				      =		set()
      UpperCAmelCase_  :	Union[str, Any]				      =		word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)    )
            UpperCAmelCase_  :	Union[str, Any]				      =		char

      UpperCAmelCase_  :	Optional[int]				      =		set(__snake_case    )
      return pairs





class   snake_case_ (lowercase__  ):




       """simple docstring"""






       _lowerCamelCase			    =				VOCAB_FILES_NAMES
       _lowerCamelCase			    =				PRETRAINED_VOCAB_FILES_MAP
       _lowerCamelCase			    =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       _lowerCamelCase			    =				CONTROL_CODES



       def __init__(     self     ,lowercase     ,lowercase     ,lowercase="<unk>"     ,**lowercase):

             """simple docstring"""





             super().__init__(unk_token=lowercase     ,**lowercase)

             with open(lowercase     ,encoding="utf-8") as vocab_handle:
                   UpperCAmelCase_  :	Optional[int]				      =		json.load(lowercase)
             UpperCAmelCase_  :	Any				      =		{v: k for k, v in self.encoder.items()}
             with open(lowercase     ,encoding="utf-8") as merges_handle:
                   UpperCAmelCase_  :	Tuple				      =		merges_handle.read().split("\n")[1:-1]
             UpperCAmelCase_  :	Tuple				      =		[tuple(merge.split()) for merge in merges]
             UpperCAmelCase_  :	str				      =		dict(zip(lowercase     ,range(len(lowercase))))
             UpperCAmelCase_  :	Optional[int]				      =		{}



       @property
       def  A_   (     self):

             """simple docstring"""





             return len(self.encoder)



       def  A_   (     self):

             """simple docstring"""





             return dict(self.encoder     ,**self.added_tokens_encoder)



       def  A_   (     self     ,lowercase):

             """simple docstring"""





             if token in self.cache:
                   return self.cache[token]
             UpperCAmelCase_  :	str				      =		tuple(lowercase)
             UpperCAmelCase_  :	List[str]				      =		tuple(list(word[:-1]) + [word[-1] + "</w>"])
             UpperCAmelCase_  :	List[str]				      =		get_pairs(lowercase)

             if not pairs:
                   return token

             while True:
                   UpperCAmelCase_  :	str				      =		min(lowercase     ,key=lambda lowercase: self.bpe_ranks.get(lowercase     ,float("inf")))
                   if bigram not in self.bpe_ranks:
                         break
                   UpperCAmelCase_      ,      UpperCAmelCase_  :	Dict				      =		bigram
                   UpperCAmelCase_  :	Dict				      =		[]
                   UpperCAmelCase_  :	str				      =		0
                   while i < len(lowercase):
                         try:
                               UpperCAmelCase_  :	str				      =		word.index(lowercase     ,lowercase)
                         except ValueError:
                               new_word.extend(word[i:])
                               break
                         else:
                               new_word.extend(word[i:j])
                               UpperCAmelCase_  :	Any				      =		j

                         if word[i] == first and i < len(lowercase) - 1 and word[i + 1] == second:
                               new_word.append(first + second)
                               i += 2
                         else:
                               new_word.append(word[i])
                               i += 1
                   UpperCAmelCase_  :	List[str]				      =		tuple(lowercase)
                   UpperCAmelCase_  :	List[str]				      =		new_word
                   if len(lowercase) == 1:
                         break
                   else:
                         UpperCAmelCase_  :	List[str]				      =		get_pairs(lowercase)
             UpperCAmelCase_  :	List[str]				      =		"@@ ".join(lowercase)
             UpperCAmelCase_  :	Tuple				      =		word[:-4]
             UpperCAmelCase_  :	Dict				      =		word
             return word



       def  A_   (     self     ,lowercase):

             """simple docstring"""





             UpperCAmelCase_  :	List[str]				      =		[]

             UpperCAmelCase_  :	Union[str, Any]				      =		re.findall(R"\S+\n?"     ,lowercase)

             for token in words:
                   split_tokens.extend(list(self.bpe(lowercase).split(" ")))
             return split_tokens



       def  A_   (     self     ,lowercase):

             """simple docstring"""





             return self.encoder.get(lowercase     ,self.encoder.get(self.unk_token))



       def  A_   (     self     ,lowercase):

             """simple docstring"""





             return self.decoder.get(lowercase     ,self.unk_token)



       def  A_   (     self     ,lowercase):

             """simple docstring"""





             UpperCAmelCase_  :	Union[str, Any]				      =		" ".join(lowercase).replace("@@ "     ,"").strip()
             return out_string



       def  A_   (     self     ,lowercase     ,lowercase = None):

             """simple docstring"""





             if not os.path.isdir(lowercase):
                   logger.error(F"""Vocabulary path ({save_directory}) should be a directory""")
                   return
             UpperCAmelCase_  :	Tuple				      =		os.path.join(
                 lowercase     ,(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])
             UpperCAmelCase_  :	Optional[int]				      =		os.path.join(
                 lowercase     ,(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"])

             with open(lowercase     ,"w"     ,encoding="utf-8") as f:
                   f.write(json.dumps(self.encoder     ,indent=2     ,sort_keys=lowercase     ,ensure_ascii=lowercase) + "\n")

             UpperCAmelCase_  :	Optional[Any]				      =		0
             with open(lowercase     ,"w"     ,encoding="utf-8") as writer:
                   writer.write("#version: 0.2\n")
                   for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     ,key=lambda lowercase: kv[1]):
                         if index != token_index:
                               logger.warning(
                                   F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                                   " Please check that the tokenizer is not corrupted!")
                               UpperCAmelCase_  :	List[str]				      =		token_index
                         writer.write(" ".join(lowercase) + "\n")
                         index += 1

             return vocab_file, merge_file

       # def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
       #     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
       #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
       #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
       #     return ''.join(tokens_generated_so_far)





import os
from argparse import ArgumentParser, Namespace

from ..data import SingleSentenceClassificationProcessor as Processor
from ..pipelines import TextClassificationPipeline
from ..utils import is_tf_available, is_torch_available, logging
from . import BaseTransformersCLICommand


if not is_tf_available() and not is_torch_available():
   raise RuntimeError('''At least one of PyTorch or TensorFlow 2.0+ should be installed to use CLI training''')

# TF training parameters
__lowerCamelCase         =			False
__lowerCamelCase         =			False







def    _snake_case (       __snake_case    )     ->     Any:






      '''simple docstring'''







      return TrainCommand(__snake_case    )





class   snake_case_ (lowercase__  ):




       """simple docstring"""






       @staticmethod
       def  A_   (     lowercase):

             """simple docstring"""





             UpperCAmelCase_  :	str				      =		parser.add_parser("train"     ,help="CLI tool to train a model on a task.")

             train_parser.add_argument(
                 "--train_data"     ,type=lowercase     ,required=lowercase     ,help="path to train (and optionally evaluation) dataset as a csv with tab separated labels and sentences."     ,)
             train_parser.add_argument(
                 "--column_label"     ,type=lowercase     ,default=0     ,help="Column of the dataset csv file with example labels.")
             train_parser.add_argument(
                 "--column_text"     ,type=lowercase     ,default=1     ,help="Column of the dataset csv file with example texts.")
             train_parser.add_argument(
                 "--column_id"     ,type=lowercase     ,default=2     ,help="Column of the dataset csv file with example ids.")
             train_parser.add_argument(
                 "--skip_first_row"     ,action="store_true"     ,help="Skip the first row of the csv file (headers).")

             train_parser.add_argument("--validation_data"     ,type=lowercase     ,default=""     ,help="path to validation dataset.")
             train_parser.add_argument(
                 "--validation_split"     ,type=lowercase     ,default=0.1     ,help="if validation dataset is not provided, fraction of train dataset to use as validation dataset."     ,)

             train_parser.add_argument("--output"     ,type=lowercase     ,default="./"     ,help="path to saved the trained model.")

             train_parser.add_argument(
                 "--task"     ,type=lowercase     ,default="text_classification"     ,help="Task to train the model on.")
             train_parser.add_argument(
                 "--model"     ,type=lowercase     ,default="bert-base-uncased"     ,help="Model's name or path to stored model.")
             train_parser.add_argument("--train_batch_size"     ,type=lowercase     ,default=32     ,help="Batch size for training.")
             train_parser.add_argument("--valid_batch_size"     ,type=lowercase     ,default=64     ,help="Batch size for validation.")
             train_parser.add_argument("--learning_rate"     ,type=lowercase     ,default=3E-5     ,help="Learning rate.")
             train_parser.add_argument("--adam_epsilon"     ,type=lowercase     ,default=1E-08     ,help="Epsilon for Adam optimizer.")
             train_parser.set_defaults(func=lowercase)



       def __init__(     self     ,lowercase):

             """simple docstring"""





             UpperCAmelCase_  :	Optional[Any]				      =		logging.get_logger("transformers-cli/training")

             UpperCAmelCase_  :	List[Any]				      =		"tf" if is_tf_available() else "torch"

             os.makedirs(args.output     ,exist_ok=lowercase)
             UpperCAmelCase_  :	Union[str, Any]				      =		args.output

             UpperCAmelCase_  :	Optional[int]				      =		args.column_label
             UpperCAmelCase_  :	Dict				      =		args.column_text
             UpperCAmelCase_  :	Optional[Any]				      =		args.column_id

             self.logger.info(F"""Loading {args.task} pipeline for {args.model}""")
             if args.task == "text_classification":
                   UpperCAmelCase_  :	List[Any]				      =		TextClassificationPipeline.from_pretrained(args.model)
             elif args.task == "token_classification":
                   raise NotImplementedError
             elif args.task == "question_answering":
                   raise NotImplementedError

             self.logger.info(F"""Loading dataset from {args.train_data}""")
             UpperCAmelCase_  :	str				      =		Processor.create_from_csv(
                 args.train_data     ,column_label=args.column_label     ,column_text=args.column_text     ,column_id=args.column_id     ,skip_first_row=args.skip_first_row     ,)
             UpperCAmelCase_  :	str				      =		None
             if args.validation_data:
                   self.logger.info(F"""Loading validation dataset from {args.validation_data}""")
                   UpperCAmelCase_  :	Union[str, Any]				      =		Processor.create_from_csv(
                       args.validation_data     ,column_label=args.column_label     ,column_text=args.column_text     ,column_id=args.column_id     ,skip_first_row=args.skip_first_row     ,)

             UpperCAmelCase_  :	List[str]				      =		args.validation_split
             UpperCAmelCase_  :	Tuple				      =		args.train_batch_size
             UpperCAmelCase_  :	Optional[int]				      =		args.valid_batch_size
             UpperCAmelCase_  :	Dict				      =		args.learning_rate
             UpperCAmelCase_  :	List[str]				      =		args.adam_epsilon



       def  A_   (     self):

             """simple docstring"""





             if self.framework == "tf":
                   return self.run_tf()
             return self.run_torch()



       def  A_   (     self):

             """simple docstring"""





             raise NotImplementedError



       def  A_   (     self):

             """simple docstring"""





             self.pipeline.fit(
                 self.train_dataset     ,validation_data=self.valid_dataset     ,validation_split=self.validation_split     ,learning_rate=self.learning_rate     ,adam_epsilon=self.adam_epsilon     ,train_batch_size=self.train_batch_size     ,valid_batch_size=self.valid_batch_size     ,)

             # Save trained pipeline
             self.pipeline.save_pretrained(self.output)
