import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__magic_name__								=logging.get_logger(__name__)

__magic_name__								={
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

__magic_name__								={
    '''vocab_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json'''},
    '''merges_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt'''},
}

__magic_name__								={
    '''ctrl''': 256,
}

__magic_name__								={
    '''Pregnancy''': 168629,
    '''Christianity''': 7675,
    '''Explain''': 106423,
    '''Fitness''': 63440,
    '''Saving''': 63163,
    '''Ask''': 27171,
    '''Ass''': 95985,
    '''Joke''': 163509,
    '''Questions''': 45622,
    '''Thoughts''': 49605,
    '''Retail''': 52342,
    '''Feminism''': 164338,
    '''Writing''': 11992,
    '''Atheism''': 192263,
    '''Netflix''': 48616,
    '''Computing''': 39639,
    '''Opinion''': 43213,
    '''Alone''': 44967,
    '''Funny''': 58917,
    '''Gaming''': 40358,
    '''Human''': 4088,
    '''India''': 1331,
    '''Joker''': 77138,
    '''Diet''': 36206,
    '''Legal''': 11859,
    '''Norman''': 4939,
    '''Tip''': 72689,
    '''Weight''': 52343,
    '''Movies''': 46273,
    '''Running''': 23425,
    '''Science''': 2090,
    '''Horror''': 37793,
    '''Confession''': 60572,
    '''Finance''': 12250,
    '''Politics''': 16360,
    '''Scary''': 191985,
    '''Support''': 12654,
    '''Technologies''': 32516,
    '''Teenage''': 66160,
    '''Event''': 32769,
    '''Learned''': 67460,
    '''Notion''': 182770,
    '''Wikipedia''': 37583,
    '''Books''': 6665,
    '''Extract''': 76050,
    '''Confessions''': 102701,
    '''Conspiracy''': 75932,
    '''Links''': 63674,
    '''Narcissus''': 150425,
    '''Relationship''': 54766,
    '''Relationships''': 134796,
    '''Reviews''': 41671,
    '''News''': 4256,
    '''Translation''': 26820,
    '''multilingual''': 128406,
}







def  __UpperCamelCase    (						A							):
     UpperCamelCase__										=      set()
     UpperCamelCase__										=      word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)							)
          UpperCamelCase__										=      char

     UpperCamelCase__										=      set(A							)
     return pairs



class     _A   (    __UpperCamelCase							):
  SCREAMING_SNAKE_CASE_	:	Tuple						=VOCAB_FILES_NAMES
  SCREAMING_SNAKE_CASE_	:	str						=PRETRAINED_VOCAB_FILES_MAP
  SCREAMING_SNAKE_CASE_	:	int						=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  SCREAMING_SNAKE_CASE_	:	Tuple						=CONTROL_CODES






  def __init__(self   ,				SCREAMING_SNAKE_CASE_   ,				SCREAMING_SNAKE_CASE_   ,				SCREAMING_SNAKE_CASE_="<unk>"   ,				**SCREAMING_SNAKE_CASE_	)		->      List[Any]:





       '''simple docstring'''





       super().__init__(unk_token=SCREAMING_SNAKE_CASE_   ,				**SCREAMING_SNAKE_CASE_	)

       with open(SCREAMING_SNAKE_CASE_   ,				encoding='''utf-8'''	) as vocab_handle:
            UpperCamelCase__										=      json.load(SCREAMING_SNAKE_CASE_	)
       UpperCamelCase__										=      {v: k for k, v in self.encoder.items()}
       with open(SCREAMING_SNAKE_CASE_   ,				encoding='''utf-8'''	) as merges_handle:
            UpperCamelCase__										=      merges_handle.read().split('''\n'''	)[1:-1]
       UpperCamelCase__										=      [tuple(merge.split()	) for merge in merges]
       UpperCamelCase__										=      dict(zip(SCREAMING_SNAKE_CASE_   ,				range(len(SCREAMING_SNAKE_CASE_	)	)	)	)
       UpperCamelCase__										=      {}






  @property
  def 		_a					(self	)		->      Optional[int]:





       '''simple docstring'''





       return len(self.encoder	)






  def 		_a					(self	)		->      Dict:





       '''simple docstring'''





       return dict(self.encoder   ,				**self.added_tokens_encoder	)






  def 		_a					(self   ,				SCREAMING_SNAKE_CASE_	)		->      List[Any]:





       '''simple docstring'''





       if token in self.cache:
            return self.cache[token]
       UpperCamelCase__										=      tuple(SCREAMING_SNAKE_CASE_	)
       UpperCamelCase__										=      tuple(list(word[:-1]	) + [word[-1] + '''</w>''']	)
       UpperCamelCase__										=      get_pairs(SCREAMING_SNAKE_CASE_	)

       if not pairs:
            return token

       while True:
            UpperCamelCase__										=      min(SCREAMING_SNAKE_CASE_   ,				key=lambda SCREAMING_SNAKE_CASE_	: self.bpe_ranks.get(SCREAMING_SNAKE_CASE_   ,				float('''inf'''	)	)	)
            if bigram not in self.bpe_ranks:
                 break
            UpperCamelCase__						,       UpperCamelCase__										=      bigram
            UpperCamelCase__										=      []
            UpperCamelCase__										=      0
            while i < len(SCREAMING_SNAKE_CASE_	):
                 try:
                      UpperCamelCase__										=      word.index(SCREAMING_SNAKE_CASE_   ,				SCREAMING_SNAKE_CASE_	)
                 except ValueError:
                      new_word.extend(word[i:]	)
                      break
                 else:
                      new_word.extend(word[i:j]	)
                      UpperCamelCase__										=      j

                 if word[i] == first and i < len(SCREAMING_SNAKE_CASE_	) - 1 and word[i + 1] == second:
                      new_word.append(first + second	)
                      i += 2
                 else:
                      new_word.append(word[i]	)
                      i += 1
            UpperCamelCase__										=      tuple(SCREAMING_SNAKE_CASE_	)
            UpperCamelCase__										=      new_word
            if len(SCREAMING_SNAKE_CASE_	) == 1:
                 break
            else:
                 UpperCamelCase__										=      get_pairs(SCREAMING_SNAKE_CASE_	)
       UpperCamelCase__										=      '''@@ '''.join(SCREAMING_SNAKE_CASE_	)
       UpperCamelCase__										=      word[:-4]
       UpperCamelCase__										=      word
       return word






  def 		_a					(self   ,				SCREAMING_SNAKE_CASE_	)		->      List[str]:





       '''simple docstring'''





       UpperCamelCase__										=      []

       UpperCamelCase__										=      re.findall(r'''\S+\n?'''   ,				SCREAMING_SNAKE_CASE_	)

       for token in words:
            split_tokens.extend(list(self.bpe(SCREAMING_SNAKE_CASE_	).split(''' '''	)	)	)
       return split_tokens






  def 		_a					(self   ,				SCREAMING_SNAKE_CASE_	)		->      Any:





       '''simple docstring'''





       return self.encoder.get(SCREAMING_SNAKE_CASE_   ,				self.encoder.get(self.unk_token	)	)






  def 		_a					(self   ,				SCREAMING_SNAKE_CASE_	)		->      List[str]:





       '''simple docstring'''





       return self.decoder.get(SCREAMING_SNAKE_CASE_   ,				self.unk_token	)






  def 		_a					(self   ,				SCREAMING_SNAKE_CASE_	)		->      List[Any]:





       '''simple docstring'''





       UpperCamelCase__										=      ''' '''.join(SCREAMING_SNAKE_CASE_	).replace('''@@ '''   ,				''''''	).strip()
       return out_string






  def 		_a					(self   ,				SCREAMING_SNAKE_CASE_   ,				SCREAMING_SNAKE_CASE_ = None	)		->      Tuple[str]:





       '''simple docstring'''





       if not os.path.isdir(SCREAMING_SNAKE_CASE_	):
            logger.error(F"Vocabulary path ({save_directory}) should be a directory"	)
            return
       UpperCamelCase__										=      os.path.join(
           SCREAMING_SNAKE_CASE_   ,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']	)
       UpperCamelCase__										=      os.path.join(
           SCREAMING_SNAKE_CASE_   ,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']	)

       with open(SCREAMING_SNAKE_CASE_   ,				'''w'''   ,				encoding='''utf-8'''	) as f:
            f.write(json.dumps(self.encoder   ,				indent=2   ,				sort_keys=SCREAMING_SNAKE_CASE_   ,				ensure_ascii=SCREAMING_SNAKE_CASE_	) + '''\n'''	)

       UpperCamelCase__										=      0
       with open(SCREAMING_SNAKE_CASE_   ,				'''w'''   ,				encoding='''utf-8'''	) as writer:
            writer.write('''#version: 0.2\n'''	)
            for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,				key=lambda SCREAMING_SNAKE_CASE_	: kv[1]	):
                 if index != token_index:
                      logger.warning(
                          F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                          ''' Please check that the tokenizer is not corrupted!'''	)
                      UpperCamelCase__										=      token_index
                 writer.write(''' '''.join(SCREAMING_SNAKE_CASE_	) + '''\n'''	)
                 index += 1

       return vocab_file, merge_file

  # def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
  #     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
  #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
  #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
  #     return ''.join(tokens_generated_so_far)




__magic_name__								={
    '''Pillow''': '''Pillow<10.0.0''',
    '''accelerate''': '''accelerate>=0.20.3''',
    '''av''': '''av==9.2.0''',
    '''beautifulsoup4''': '''beautifulsoup4''',
    '''black''': '''black~=23.1''',
    '''codecarbon''': '''codecarbon==1.2.0''',
    '''cookiecutter''': '''cookiecutter==1.7.3''',
    '''dataclasses''': '''dataclasses''',
    '''datasets''': '''datasets!=2.5.0''',
    '''decord''': '''decord==0.6.0''',
    '''deepspeed''': '''deepspeed>=0.9.3''',
    '''diffusers''': '''diffusers''',
    '''dill''': '''dill<0.3.5''',
    '''evaluate''': '''evaluate>=0.2.0''',
    '''fairscale''': '''fairscale>0.3''',
    '''faiss-cpu''': '''faiss-cpu''',
    '''fastapi''': '''fastapi''',
    '''filelock''': '''filelock''',
    '''flax''': '''flax>=0.4.1,<=0.7.0''',
    '''ftfy''': '''ftfy''',
    '''fugashi''': '''fugashi>=1.0''',
    '''GitPython''': '''GitPython<3.1.19''',
    '''hf-doc-builder''': '''hf-doc-builder>=0.3.0''',
    '''huggingface-hub''': '''huggingface-hub>=0.14.1,<1.0''',
    '''importlib_metadata''': '''importlib_metadata''',
    '''ipadic''': '''ipadic>=1.0.0,<2.0''',
    '''isort''': '''isort>=5.5.4''',
    '''jax''': '''jax>=0.2.8,!=0.3.2,<=0.4.13''',
    '''jaxlib''': '''jaxlib>=0.1.65,<=0.4.13''',
    '''jieba''': '''jieba''',
    '''kenlm''': '''kenlm''',
    '''keras-nlp''': '''keras-nlp>=0.3.1''',
    '''librosa''': '''librosa''',
    '''nltk''': '''nltk''',
    '''natten''': '''natten>=0.14.6''',
    '''numpy''': '''numpy>=1.17''',
    '''onnxconverter-common''': '''onnxconverter-common''',
    '''onnxruntime-tools''': '''onnxruntime-tools>=1.4.2''',
    '''onnxruntime''': '''onnxruntime>=1.4.0''',
    '''opencv-python''': '''opencv-python''',
    '''optuna''': '''optuna''',
    '''optax''': '''optax>=0.0.8,<=0.1.4''',
    '''packaging''': '''packaging>=20.0''',
    '''parameterized''': '''parameterized''',
    '''phonemizer''': '''phonemizer''',
    '''protobuf''': '''protobuf''',
    '''psutil''': '''psutil''',
    '''pyyaml''': '''pyyaml>=5.1''',
    '''pydantic''': '''pydantic<2''',
    '''pytest''': '''pytest>=7.2.0''',
    '''pytest-timeout''': '''pytest-timeout''',
    '''pytest-xdist''': '''pytest-xdist''',
    '''python''': '''python>=3.8.0''',
    '''ray[tune]''': '''ray[tune]''',
    '''regex''': '''regex!=2019.12.17''',
    '''requests''': '''requests''',
    '''rhoknp''': '''rhoknp>=1.1.0,<1.3.1''',
    '''rjieba''': '''rjieba''',
    '''rouge-score''': '''rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1''',
    '''ruff''': '''ruff>=0.0.241,<=0.0.259''',
    '''sacrebleu''': '''sacrebleu>=1.4.12,<2.0.0''',
    '''sacremoses''': '''sacremoses''',
    '''safetensors''': '''safetensors>=0.3.1''',
    '''sagemaker''': '''sagemaker>=2.31.0''',
    '''scikit-learn''': '''scikit-learn''',
    '''sentencepiece''': '''sentencepiece>=0.1.91,!=0.1.92''',
    '''sigopt''': '''sigopt''',
    '''starlette''': '''starlette''',
    '''sudachipy''': '''sudachipy>=0.6.6''',
    '''sudachidict_core''': '''sudachidict_core>=20220729''',
    '''tensorflow-cpu''': '''tensorflow-cpu>=2.6,<2.14''',
    '''tensorflow''': '''tensorflow>=2.6,<2.14''',
    '''tensorflow-text''': '''tensorflow-text<2.14''',
    '''tf2onnx''': '''tf2onnx''',
    '''timeout-decorator''': '''timeout-decorator''',
    '''timm''': '''timm''',
    '''tokenizers''': '''tokenizers>=0.11.1,!=0.11.3,<0.14''',
    '''torch''': '''torch>=1.9,!=1.12.0''',
    '''torchaudio''': '''torchaudio''',
    '''torchvision''': '''torchvision''',
    '''pyctcdecode''': '''pyctcdecode>=0.4.0''',
    '''tqdm''': '''tqdm>=4.27''',
    '''unidic''': '''unidic>=1.0.2''',
    '''unidic_lite''': '''unidic_lite>=1.0.7''',
    '''urllib3''': '''urllib3<2.0.0''',
    '''uvicorn''': '''uvicorn''',
}



