
import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCamelCase__    :		Tuple          =   logging.get_logger(__name__)

lowerCamelCase__    :		Tuple          =   {
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

lowerCamelCase__    :		Optional[Any]          =   {
    """vocab_file""": {"""ctrl""": """https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json"""},
    """merges_file""": {"""ctrl""": """https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt"""},
}

lowerCamelCase__    :		List[Any]          =   {
    """ctrl""": 2_5_6,
}

lowerCamelCase__    :		Union[str, Any]          =   {
    """Pregnancy""": 1_6_8_6_2_9,
    """Christianity""": 7_6_7_5,
    """Explain""": 1_0_6_4_2_3,
    """Fitness""": 6_3_4_4_0,
    """Saving""": 6_3_1_6_3,
    """Ask""": 2_7_1_7_1,
    """Ass""": 9_5_9_8_5,
    """Joke""": 1_6_3_5_0_9,
    """Questions""": 4_5_6_2_2,
    """Thoughts""": 4_9_6_0_5,
    """Retail""": 5_2_3_4_2,
    """Feminism""": 1_6_4_3_3_8,
    """Writing""": 1_1_9_9_2,
    """Atheism""": 1_9_2_2_6_3,
    """Netflix""": 4_8_6_1_6,
    """Computing""": 3_9_6_3_9,
    """Opinion""": 4_3_2_1_3,
    """Alone""": 4_4_9_6_7,
    """Funny""": 5_8_9_1_7,
    """Gaming""": 4_0_3_5_8,
    """Human""": 4_0_8_8,
    """India""": 1_3_3_1,
    """Joker""": 7_7_1_3_8,
    """Diet""": 3_6_2_0_6,
    """Legal""": 1_1_8_5_9,
    """Norman""": 4_9_3_9,
    """Tip""": 7_2_6_8_9,
    """Weight""": 5_2_3_4_3,
    """Movies""": 4_6_2_7_3,
    """Running""": 2_3_4_2_5,
    """Science""": 2_0_9_0,
    """Horror""": 3_7_7_9_3,
    """Confession""": 6_0_5_7_2,
    """Finance""": 1_2_2_5_0,
    """Politics""": 1_6_3_6_0,
    """Scary""": 1_9_1_9_8_5,
    """Support""": 1_2_6_5_4,
    """Technologies""": 3_2_5_1_6,
    """Teenage""": 6_6_1_6_0,
    """Event""": 3_2_7_6_9,
    """Learned""": 6_7_4_6_0,
    """Notion""": 1_8_2_7_7_0,
    """Wikipedia""": 3_7_5_8_3,
    """Books""": 6_6_6_5,
    """Extract""": 7_6_0_5_0,
    """Confessions""": 1_0_2_7_0_1,
    """Conspiracy""": 7_5_9_3_2,
    """Links""": 6_3_6_7_4,
    """Narcissus""": 1_5_0_4_2_5,
    """Relationship""": 5_4_7_6_6,
    """Relationships""": 1_3_4_7_9_6,
    """Reviews""": 4_1_6_7_1,
    """News""": 4_2_5_6,
    """Translation""": 2_6_8_2_0,
    """multilingual""": 1_2_8_4_0_6,
}

def 		UpperCamelCase	(      lowercase_   )		->	List[Any]:


  '''simple docstring'''






  lowercase__  :   Any    		=      set()
  lowercase__  :   List[str]    		=      word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)   )
    lowercase__  :   Dict    		=      char

  lowercase__  :   Union[str, Any]    		=      set(UpperCamelCase__   )
  return pairs



class       _snake_case   (	lowercase_							):
 __lowerCAmelCase  :     Optional[int]        =			VOCAB_FILES_NAMES
 __lowerCAmelCase  :     Tuple        =			PRETRAINED_VOCAB_FILES_MAP
 __lowerCAmelCase  :     Union[str, Any]        =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __lowerCAmelCase  :     Tuple        =			CONTROL_CODES


 def __init__(					self					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_="<unk>"					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   super().__init__(unk_token=SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_)

   with open(SCREAMING_SNAKE_CASE_					,  encoding="""utf-8""") as vocab_handle:
     lowercase__  :   List[Any]    		=      json.load(SCREAMING_SNAKE_CASE_)
   lowercase__  :   Optional[int]    		=      {v: k for k, v in self.encoder.items()}
   with open(SCREAMING_SNAKE_CASE_					,  encoding="""utf-8""") as merges_handle:
     lowercase__  :   Optional[int]    		=      merges_handle.read().split("""\n""")[1:-1]
   lowercase__  :   Optional[Any]    		=      [tuple(merge.split()) for merge in merges]
   lowercase__  :   str    		=      dict(zip(SCREAMING_SNAKE_CASE_					,  range(len(SCREAMING_SNAKE_CASE_))))
   lowercase__  :   Any    		=      {}


 @property
 def      lowercase__     (					self):


   '''simple docstring'''




   return len(self.encoder)


 def      lowercase__     (					self):


   '''simple docstring'''




   return dict(self.encoder					,  **self.added_tokens_encoder)


 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   if token in self.cache:
     return self.cache[token]
   lowercase__  :   str    		=      tuple(SCREAMING_SNAKE_CASE_)
   lowercase__  :   List[str]    		=      tuple(list(word[:-1]) + [word[-1] + """</w>"""])
   lowercase__  :   int    		=      get_pairs(SCREAMING_SNAKE_CASE_)

   if not pairs:
     return token

   while True:
     lowercase__  :   Tuple    		=      min(SCREAMING_SNAKE_CASE_					,  key=lambda SCREAMING_SNAKE_CASE_: self.bpe_ranks.get(SCREAMING_SNAKE_CASE_					,  float("""inf""")))
     if bigram not in self.bpe_ranks:
       break
     lowercase__		,    lowercase__  :   List[str]    		=      bigram
     lowercase__  :   int    		=      []
     lowercase__  :   Tuple    		=      0
     while i < len(SCREAMING_SNAKE_CASE_):
       try:
         lowercase__  :   Tuple    		=      word.index(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)
       except ValueError:
         new_word.extend(word[i:])
         break
       else:
         new_word.extend(word[i:j])
         lowercase__  :   Tuple    		=      j

       if word[i] == first and i < len(SCREAMING_SNAKE_CASE_) - 1 and word[i + 1] == second:
         new_word.append(first + second)
         i += 2
       else:
         new_word.append(word[i])
         i += 1
     lowercase__  :   Union[str, Any]    		=      tuple(SCREAMING_SNAKE_CASE_)
     lowercase__  :   Union[str, Any]    		=      new_word
     if len(SCREAMING_SNAKE_CASE_) == 1:
       break
     else:
       lowercase__  :   List[Any]    		=      get_pairs(SCREAMING_SNAKE_CASE_)
   lowercase__  :   Optional[int]    		=      """@@ """.join(SCREAMING_SNAKE_CASE_)
   lowercase__  :   int    		=      word[:-4]
   lowercase__  :   int    		=      word
   return word


 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   lowercase__  :   Dict    		=      []

   lowercase__  :   str    		=      re.findall(R"""\S+\n?"""					,  SCREAMING_SNAKE_CASE_)

   for token in words:
     split_tokens.extend(list(self.bpe(SCREAMING_SNAKE_CASE_).split(""" """)))
   return split_tokens


 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   return self.encoder.get(SCREAMING_SNAKE_CASE_					,  self.encoder.get(self.unk_token))


 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   return self.decoder.get(SCREAMING_SNAKE_CASE_					,  self.unk_token)


 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   lowercase__  :   Optional[Any]    		=      """ """.join(SCREAMING_SNAKE_CASE_).replace("""@@ """					,  """""").strip()
   return out_string



 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_ = None):


   '''simple docstring'''




   if not os.path.isdir(SCREAMING_SNAKE_CASE_):
     logger.error(f'Vocabulary path ({save_directory}) should be a directory')
     return
   lowercase__  :   List[str]    		=      os.path.join(
       SCREAMING_SNAKE_CASE_					,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""])
   lowercase__  :   int    		=      os.path.join(
       SCREAMING_SNAKE_CASE_					,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""])

   with open(SCREAMING_SNAKE_CASE_					,  """w"""					,  encoding="""utf-8""") as f:
     f.write(json.dumps(self.encoder					,  indent=2					,  sort_keys=SCREAMING_SNAKE_CASE_					,  ensure_ascii=SCREAMING_SNAKE_CASE_) + """\n""")

   lowercase__  :   Tuple    		=      0
   with open(SCREAMING_SNAKE_CASE_					,  """w"""					,  encoding="""utf-8""") as writer:
     writer.write("""#version: 0.2\n""")
     for bpe_tokens, token_index in sorted(self.bpe_ranks.items()					,  key=lambda SCREAMING_SNAKE_CASE_: kv[1]):
       if index != token_index:
         logger.warning(
             f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
             """ Please check that the tokenizer is not corrupted!""")
         lowercase__  :   Optional[int]    		=      token_index
       writer.write(""" """.join(SCREAMING_SNAKE_CASE_) + """\n""")
       index += 1

   return vocab_file, merge_file

 # def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
 #     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
 #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
 #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
 #     return ''.join(tokens_generated_so_far)








from ..utils import DummyObject, requires_backends



class       _snake_case   (	metaclass=UpperCAmelCase_							):
 __lowerCAmelCase  :     Any        =			['flax']


 def __init__(					self					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(self					,  ["""flax"""])


 @classmethod
 def      lowercase__     (					cls					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(cls					,  ["""flax"""])



 @classmethod
 def      lowercase__     (					cls					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(cls					,  ["""flax"""])



class       _snake_case   (	metaclass=UpperCAmelCase_							):
 __lowerCAmelCase  :     Any        =			['flax']


 def __init__(					self					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(self					,  ["""flax"""])


 @classmethod
 def      lowercase__     (					cls					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(cls					,  ["""flax"""])



 @classmethod
 def      lowercase__     (					cls					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(cls					,  ["""flax"""])



class       _snake_case   (	metaclass=UpperCAmelCase_							):
 __lowerCAmelCase  :     Union[str, Any]        =			['flax']


 def __init__(					self					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(self					,  ["""flax"""])


 @classmethod
 def      lowercase__     (					cls					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(cls					,  ["""flax"""])



 @classmethod
 def      lowercase__     (					cls					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(cls					,  ["""flax"""])



class       _snake_case   (	metaclass=UpperCAmelCase_							):
 __lowerCAmelCase  :     int        =			['flax']


 def __init__(					self					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(self					,  ["""flax"""])


 @classmethod
 def      lowercase__     (					cls					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(cls					,  ["""flax"""])



 @classmethod
 def      lowercase__     (					cls					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(cls					,  ["""flax"""])



class       _snake_case   (	metaclass=UpperCAmelCase_							):
 __lowerCAmelCase  :     Optional[Any]        =			['flax']


 def __init__(					self					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(self					,  ["""flax"""])


 @classmethod
 def      lowercase__     (					cls					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(cls					,  ["""flax"""])



 @classmethod
 def      lowercase__     (					cls					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(cls					,  ["""flax"""])



class       _snake_case   (	metaclass=UpperCAmelCase_							):
 __lowerCAmelCase  :     Tuple        =			['flax']


 def __init__(					self					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(self					,  ["""flax"""])


 @classmethod
 def      lowercase__     (					cls					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(cls					,  ["""flax"""])



 @classmethod
 def      lowercase__     (					cls					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(cls					,  ["""flax"""])



class       _snake_case   (	metaclass=UpperCAmelCase_							):
 __lowerCAmelCase  :     List[str]        =			['flax']


 def __init__(					self					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(self					,  ["""flax"""])


 @classmethod
 def      lowercase__     (					cls					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(cls					,  ["""flax"""])



 @classmethod
 def      lowercase__     (					cls					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(cls					,  ["""flax"""])



class       _snake_case   (	metaclass=UpperCAmelCase_							):
 __lowerCAmelCase  :     Union[str, Any]        =			['flax']


 def __init__(					self					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(self					,  ["""flax"""])


 @classmethod
 def      lowercase__     (					cls					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(cls					,  ["""flax"""])



 @classmethod
 def      lowercase__     (					cls					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(cls					,  ["""flax"""])



class       _snake_case   (	metaclass=UpperCAmelCase_							):
 __lowerCAmelCase  :     Optional[Any]        =			['flax']


 def __init__(					self					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(self					,  ["""flax"""])


 @classmethod
 def      lowercase__     (					cls					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(cls					,  ["""flax"""])



 @classmethod
 def      lowercase__     (					cls					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(cls					,  ["""flax"""])



class       _snake_case   (	metaclass=UpperCAmelCase_							):
 __lowerCAmelCase  :     Dict        =			['flax']


 def __init__(					self					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(self					,  ["""flax"""])


 @classmethod
 def      lowercase__     (					cls					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(cls					,  ["""flax"""])



 @classmethod
 def      lowercase__     (					cls					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(cls					,  ["""flax"""])



class       _snake_case   (	metaclass=UpperCAmelCase_							):
 __lowerCAmelCase  :     Optional[Any]        =			['flax']


 def __init__(					self					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(self					,  ["""flax"""])


 @classmethod
 def      lowercase__     (					cls					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(cls					,  ["""flax"""])



 @classmethod
 def      lowercase__     (					cls					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(cls					,  ["""flax"""])



class       _snake_case   (	metaclass=UpperCAmelCase_							):
 __lowerCAmelCase  :     Optional[Any]        =			['flax']


 def __init__(					self					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(self					,  ["""flax"""])


 @classmethod
 def      lowercase__     (					cls					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(cls					,  ["""flax"""])



 @classmethod
 def      lowercase__     (					cls					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(cls					,  ["""flax"""])



class       _snake_case   (	metaclass=UpperCAmelCase_							):
 __lowerCAmelCase  :     List[Any]        =			['flax']


 def __init__(					self					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(self					,  ["""flax"""])


 @classmethod
 def      lowercase__     (					cls					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(cls					,  ["""flax"""])



 @classmethod
 def      lowercase__     (					cls					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   requires_backends(cls					,  ["""flax"""])






