
'''simple docstring'''



import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCamelCase__      :		List[str]       							=  logging.get_logger(__name__)

UpperCamelCase__      :		str       							=  {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

UpperCamelCase__      :		Any       							=  {
    "vocab_file": {"ctrl": "https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json"},
    "merges_file": {"ctrl": "https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt"},
}

UpperCamelCase__      :		Any       							=  {
    "ctrl": 256,
}

UpperCamelCase__      :		Optional[int]       							=  {
    "Pregnancy": 168_629,
    "Christianity": 7_675,
    "Explain": 106_423,
    "Fitness": 63_440,
    "Saving": 63_163,
    "Ask": 27_171,
    "Ass": 95_985,
    "Joke": 163_509,
    "Questions": 45_622,
    "Thoughts": 49_605,
    "Retail": 52_342,
    "Feminism": 164_338,
    "Writing": 11_992,
    "Atheism": 192_263,
    "Netflix": 48_616,
    "Computing": 39_639,
    "Opinion": 43_213,
    "Alone": 44_967,
    "Funny": 58_917,
    "Gaming": 40_358,
    "Human": 4_088,
    "India": 1_331,
    "Joker": 77_138,
    "Diet": 36_206,
    "Legal": 11_859,
    "Norman": 4_939,
    "Tip": 72_689,
    "Weight": 52_343,
    "Movies": 46_273,
    "Running": 23_425,
    "Science": 2_090,
    "Horror": 37_793,
    "Confession": 60_572,
    "Finance": 12_250,
    "Politics": 16_360,
    "Scary": 191_985,
    "Support": 12_654,
    "Technologies": 32_516,
    "Teenage": 66_160,
    "Event": 32_769,
    "Learned": 67_460,
    "Notion": 182_770,
    "Wikipedia": 37_583,
    "Books": 6_665,
    "Extract": 76_050,
    "Confessions": 102_701,
    "Conspiracy": 75_932,
    "Links": 63_674,
    "Narcissus": 150_425,
    "Relationship": 54_766,
    "Relationships": 134_796,
    "Reviews": 41_671,
    "News": 4_256,
    "Translation": 26_820,
    "multilingual": 128_406,
}
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     Dict:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				set()
   _SCREAMING_SNAKE_CASE      =				word[0]
   for char in word[1:]:
      pairs.add((prev_char, char)						)
      _SCREAMING_SNAKE_CASE      =				char

   _SCREAMING_SNAKE_CASE      =				set(SCREAMING_SNAKE_CASE_						)
   return pairs







class    _a				(_lowerCamelCase):





       """simple docstring"""




       SCREAMING_SNAKE_CASE      =  VOCAB_FILES_NAMES
       SCREAMING_SNAKE_CASE      =  PRETRAINED_VOCAB_FILES_MAP
       SCREAMING_SNAKE_CASE      =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       SCREAMING_SNAKE_CASE      =  CONTROL_CODES






       def __init__(							self       ,							A__       ,							A__       ,							A__="<unk>"       ,							**A__      )    ->			Union[str, Any]:
          super().__init__(unk_token=A__       ,							**A__      )

          with open(A__       ,							encoding="""utf-8"""      ) as vocab_handle:
             _SCREAMING_SNAKE_CASE      =				json.load(A__      )
          _SCREAMING_SNAKE_CASE      =				{v: k for k, v in self.encoder.items()}
          with open(A__       ,							encoding="""utf-8"""      ) as merges_handle:
             _SCREAMING_SNAKE_CASE      =				merges_handle.read().split("""\n"""      )[1:-1]
          _SCREAMING_SNAKE_CASE      =				[tuple(merge.split()      ) for merge in merges]
          _SCREAMING_SNAKE_CASE      =				dict(zip(A__       ,							range(len(A__      )      )      )      )
          _SCREAMING_SNAKE_CASE      =				{}






       @property
       def        UpperCamelCase				(							self      )    ->			str:
          return len(self.encoder      )






       def        UpperCamelCase				(							self      )    ->			Any:
          return dict(self.encoder       ,							**self.added_tokens_encoder      )






       def        UpperCamelCase				(							self       ,							A__      )    ->			int:
          if token in self.cache:
             return self.cache[token]
          _SCREAMING_SNAKE_CASE      =				tuple(A__      )
          _SCREAMING_SNAKE_CASE      =				tuple(list(word[:-1]      ) + [word[-1] + """</w>"""]      )
          _SCREAMING_SNAKE_CASE      =				get_pairs(A__      )

          if not pairs:
             return token

          while True:
             _SCREAMING_SNAKE_CASE      =				min(A__       ,							key=lambda A__      : self.bpe_ranks.get(A__       ,							float("""inf"""      )      )      )
             if bigram not in self.bpe_ranks:
                break
             _SCREAMING_SNAKE_CASE ,							_SCREAMING_SNAKE_CASE      =				bigram
             _SCREAMING_SNAKE_CASE      =				[]
             _SCREAMING_SNAKE_CASE      =				0
             while i < len(A__      ):
                try:
                   _SCREAMING_SNAKE_CASE      =				word.index(A__       ,							A__      )
                except ValueError:
                   new_word.extend(word[i:]      )
                   break
                else:
                   new_word.extend(word[i:j]      )
                   _SCREAMING_SNAKE_CASE      =				j

                if word[i] == first and i < len(A__      ) - 1 and word[i + 1] == second:
                   new_word.append(first + second      )
                   i += 2
                else:
                   new_word.append(word[i]      )
                   i += 1
             _SCREAMING_SNAKE_CASE      =				tuple(A__      )
             _SCREAMING_SNAKE_CASE      =				new_word
             if len(A__      ) == 1:
                break
             else:
                _SCREAMING_SNAKE_CASE      =				get_pairs(A__      )
          _SCREAMING_SNAKE_CASE      =				"""@@ """.join(A__      )
          _SCREAMING_SNAKE_CASE      =				word[:-4]
          _SCREAMING_SNAKE_CASE      =				word
          return word






       def        UpperCamelCase				(							self       ,							A__      )    ->			Dict:
          _SCREAMING_SNAKE_CASE      =				[]

          _SCREAMING_SNAKE_CASE      =				re.findall(R"""\S+\n?"""       ,							A__      )

          for token in words:
             split_tokens.extend(list(self.bpe(A__      ).split(""" """      )      )      )
          return split_tokens






       def        UpperCamelCase				(							self       ,							A__      )    ->			List[Any]:
          return self.encoder.get(A__       ,							self.encoder.get(self.unk_token      )      )






       def        UpperCamelCase				(							self       ,							A__      )    ->			Tuple:
          return self.decoder.get(A__       ,							self.unk_token      )






       def        UpperCamelCase				(							self       ,							A__      )    ->			List[str]:
          _SCREAMING_SNAKE_CASE      =				""" """.join(A__      ).replace("""@@ """       ,							""""""      ).strip()
          return out_string






       def        UpperCamelCase				(							self       ,							A__       ,							A__ = None      )    ->			Tuple[str]:
          if not os.path.isdir(A__      ):
             logger.error(F"Vocabulary path ({save_directory}) should be a directory"      )
             return
          _SCREAMING_SNAKE_CASE      =				os.path.join(
              A__       ,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]      )
          _SCREAMING_SNAKE_CASE      =				os.path.join(
              A__       ,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]      )

          with open(A__       ,							"""w"""       ,							encoding="""utf-8"""      ) as f:
             f.write(json.dumps(self.encoder       ,							indent=2       ,							sort_keys=A__       ,							ensure_ascii=A__      ) + """\n"""      )

          _SCREAMING_SNAKE_CASE      =				0
          with open(A__       ,							"""w"""       ,							encoding="""utf-8"""      ) as writer:
             writer.write("""#version: 0.2\n"""      )
             for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,							key=lambda A__      : kv[1]      ):
                if index != token_index:
                   logger.warning(
                       F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                       """ Please check that the tokenizer is not corrupted!"""      )
                   _SCREAMING_SNAKE_CASE      =				token_index
                writer.write(""" """.join(A__      ) + """\n"""      )
                index += 1

          return vocab_file, merge_file

       # def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
       #     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
       #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
       #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
       #     return ''.join(tokens_generated_so_far)








'''simple docstring'''



import pytest
import requests

from datasets.utils.file_utils import http_head

from .utils import OfflineSimulationMode, RequestWouldHangIndefinitelyError, offline
@pytest.mark.integration
def    lowerCAmelCase_	(				)     ->     List[Any]:




   """simple docstring"""
   with offline(OfflineSimulationMode.CONNECTION_TIMES_OUT						):
      with pytest.raises(SCREAMING_SNAKE_CASE_						):
         requests.request("""GET"""    ,      """https://huggingface.co"""						)
      with pytest.raises(requests.exceptions.ConnectTimeout						):
         requests.request("""GET"""    ,      """https://huggingface.co"""    ,      timeout=1.0						)
@pytest.mark.integration
def    lowerCAmelCase_	(				)     ->     int:




   """simple docstring"""
   with offline(OfflineSimulationMode.CONNECTION_FAILS						):
      with pytest.raises(requests.exceptions.ConnectionError						):
         requests.request("""GET"""    ,      """https://huggingface.co"""						)







def    lowerCAmelCase_	(				)     ->     Optional[Any]:




   """simple docstring"""
   with offline(OfflineSimulationMode.HF_DATASETS_OFFLINE_SET_TO_1						):
      with pytest.raises(SCREAMING_SNAKE_CASE_						):
         http_head("""https://huggingface.co"""						)






