





'''simple docstring'''

import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


a        =						logging.get_logger(__name__)


a        =						{"vocab_file": "vocab.json", "merges_file": "merges.txt"}

a        =						{
    "vocab_file": {
        "allenai/longformer-base-4096": "https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json",
        "allenai/longformer-large-4096": (
            "https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"
        ),
        "allenai/longformer-large-4096-finetuned-triviaqa": (
            "https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"
        ),
        "allenai/longformer-base-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"
        ),
        "allenai/longformer-large-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"
        ),
    },
    "merges_file": {
        "allenai/longformer-base-4096": "https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt",
        "allenai/longformer-large-4096": (
            "https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"
        ),
        "allenai/longformer-large-4096-finetuned-triviaqa": (
            "https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"
        ),
        "allenai/longformer-base-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"
        ),
        "allenai/longformer-large-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"
        ),
    },
}

a        =						{
    "allenai/longformer-base-4096": 4096,
    "allenai/longformer-large-4096": 4096,
    "allenai/longformer-large-4096-finetuned-triviaqa": 4096,
    "allenai/longformer-base-4096-extra.pos.embd.only": 4096,
    "allenai/longformer-large-4096-extra.pos.embd.only": 4096,
}

@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def 						__magic_name__       (							)						->					str:

   '''simple docstring'''







   __SCREAMING_SNAKE_CASE      							=       (
       list(range(ord("""!"""      )							,	ord("""~"""      ) + 1      )      ) + list(range(ord("""¡"""      )							,	ord("""¬"""      ) + 1      )      ) + list(range(ord("""®"""      )							,	ord("""ÿ"""      ) + 1      )      )
   )
   __SCREAMING_SNAKE_CASE      							=       bs[:]
   __SCREAMING_SNAKE_CASE      							=       0
   for b in range(2**8      ):
      if b not in bs:
         bs.append(__UpperCAmelCase      )
         cs.append(2**8 + n      )
         n += 1
   __SCREAMING_SNAKE_CASE      							=       [chr(__UpperCAmelCase      ) for n in cs]
   return dict(zip(__UpperCAmelCase							,	__UpperCAmelCase      )      )

def 						__magic_name__       (							__UpperCAmelCase      )						->					int:

   '''simple docstring'''







   __SCREAMING_SNAKE_CASE      							=       set()
   __SCREAMING_SNAKE_CASE      							=       word[0]
   for char in word[1:]:
      pairs.add((prev_char, char)      )
      __SCREAMING_SNAKE_CASE      							=       char
   return pairs







class __a (   _snake_case			):
   __UpperCamelCase   : Dict	      =	VOCAB_FILES_NAMES
   __UpperCamelCase   : Dict	      =	PRETRAINED_VOCAB_FILES_MAP
   __UpperCamelCase   : Optional[Any]	      =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __UpperCamelCase   : List[str]	      =	['input_ids', 'attention_mask']





   def __init__(					self				:			Optional[int]				,lowerCamelCase				:			List[Any]				,lowerCamelCase				:			List[Any]				,lowerCamelCase				:			Tuple="replace"				,lowerCamelCase				:			Tuple="<s>"				,lowerCamelCase				:			List[Any]="</s>"				,lowerCamelCase				:			Optional[Any]="</s>"				,lowerCamelCase				:			Optional[int]="<s>"				,lowerCamelCase				:			Optional[int]="<unk>"				,lowerCamelCase				:			List[Any]="<pad>"				,lowerCamelCase				:			Dict="<mask>"				,lowerCamelCase				:			List[str]=False				,**lowerCamelCase				:			Any				,):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       AddedToken(lowerCamelCase				,lstrip=lowerCamelCase				,rstrip=lowerCamelCase    ) if isinstance(lowerCamelCase				,lowerCamelCase    ) else bos_token
      __SCREAMING_SNAKE_CASE      							=       AddedToken(lowerCamelCase				,lstrip=lowerCamelCase				,rstrip=lowerCamelCase    ) if isinstance(lowerCamelCase				,lowerCamelCase    ) else eos_token
      __SCREAMING_SNAKE_CASE      							=       AddedToken(lowerCamelCase				,lstrip=lowerCamelCase				,rstrip=lowerCamelCase    ) if isinstance(lowerCamelCase				,lowerCamelCase    ) else sep_token
      __SCREAMING_SNAKE_CASE      							=       AddedToken(lowerCamelCase				,lstrip=lowerCamelCase				,rstrip=lowerCamelCase    ) if isinstance(lowerCamelCase				,lowerCamelCase    ) else cls_token
      __SCREAMING_SNAKE_CASE      							=       AddedToken(lowerCamelCase				,lstrip=lowerCamelCase				,rstrip=lowerCamelCase    ) if isinstance(lowerCamelCase				,lowerCamelCase    ) else unk_token
      __SCREAMING_SNAKE_CASE      							=       AddedToken(lowerCamelCase				,lstrip=lowerCamelCase				,rstrip=lowerCamelCase    ) if isinstance(lowerCamelCase				,lowerCamelCase    ) else pad_token

      # Mask token behave like a normal word, i.e. include the space before it
      __SCREAMING_SNAKE_CASE      							=       AddedToken(lowerCamelCase				,lstrip=lowerCamelCase				,rstrip=lowerCamelCase    ) if isinstance(lowerCamelCase				,lowerCamelCase    ) else mask_token

      super().__init__(
          errors=lowerCamelCase				,bos_token=lowerCamelCase				,eos_token=lowerCamelCase				,unk_token=lowerCamelCase				,sep_token=lowerCamelCase				,cls_token=lowerCamelCase				,pad_token=lowerCamelCase				,mask_token=lowerCamelCase				,add_prefix_space=lowerCamelCase				,**lowerCamelCase				,)

      with open(lowerCamelCase				,encoding="""utf-8"""    ) as vocab_handle:
         __SCREAMING_SNAKE_CASE      							=       json.load(lowerCamelCase    )
      __SCREAMING_SNAKE_CASE      							=       {v: k for k, v in self.encoder.items()}
      __SCREAMING_SNAKE_CASE      							=       errors  # how to handle errors in decoding
      __SCREAMING_SNAKE_CASE      							=       bytes_to_unicode()
      __SCREAMING_SNAKE_CASE      							=       {v: k for k, v in self.byte_encoder.items()}
      with open(lowerCamelCase				,encoding="""utf-8"""    ) as merges_handle:
         __SCREAMING_SNAKE_CASE      							=       merges_handle.read().split("""\n"""    )[1:-1]
      __SCREAMING_SNAKE_CASE      							=       [tuple(merge.split()    ) for merge in bpe_merges]
      __SCREAMING_SNAKE_CASE      							=       dict(zip(lowerCamelCase				,range(len(lowerCamelCase    )    )    )    )
      __SCREAMING_SNAKE_CASE      							=       {}
      __SCREAMING_SNAKE_CASE      							=       add_prefix_space

      # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
      __SCREAMING_SNAKE_CASE      							=       re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""    )





   @property
   def      UpperCAmelCase__     (					self				:			List[Any]    ):

      '''simple docstring'''

      return len(self.encoder    )





   def      UpperCAmelCase__     (					self				:			Dict    ):

      '''simple docstring'''

      return dict(self.encoder				,**self.added_tokens_encoder    )





   def      UpperCAmelCase__     (					self				:			Optional[int]				,lowerCamelCase				:			List[Any]    ):

      '''simple docstring'''

      if token in self.cache:
         return self.cache[token]
      __SCREAMING_SNAKE_CASE      							=       tuple(lowerCamelCase    )
      __SCREAMING_SNAKE_CASE      							=       get_pairs(lowerCamelCase    )

      if not pairs:
         return token

      while True:
         __SCREAMING_SNAKE_CASE      							=       min(lowerCamelCase				,key=lambda lowerCamelCase    : self.bpe_ranks.get(lowerCamelCase				,float("""inf"""    )    )    )
         if bigram not in self.bpe_ranks:
            break
         __SCREAMING_SNAKE_CASE							,     __SCREAMING_SNAKE_CASE      							=       bigram
         __SCREAMING_SNAKE_CASE      							=       []
         __SCREAMING_SNAKE_CASE      							=       0
         while i < len(lowerCamelCase    ):
            try:
               __SCREAMING_SNAKE_CASE      							=       word.index(lowerCamelCase				,lowerCamelCase    )
            except ValueError:
               new_word.extend(word[i:]    )
               break
            else:
               new_word.extend(word[i:j]    )
               __SCREAMING_SNAKE_CASE      							=       j

            if word[i] == first and i < len(lowerCamelCase    ) - 1 and word[i + 1] == second:
               new_word.append(first + second    )
               i += 2
            else:
               new_word.append(word[i]    )
               i += 1
         __SCREAMING_SNAKE_CASE      							=       tuple(lowerCamelCase    )
         __SCREAMING_SNAKE_CASE      							=       new_word
         if len(lowerCamelCase    ) == 1:
            break
         else:
            __SCREAMING_SNAKE_CASE      							=       get_pairs(lowerCamelCase    )
      __SCREAMING_SNAKE_CASE      							=       """ """.join(lowerCamelCase    )
      __SCREAMING_SNAKE_CASE      							=       word
      return word





   def      UpperCAmelCase__     (					self				:			List[str]				,lowerCamelCase				:			Union[str, Any]    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       []
      for token in re.findall(self.pat				,lowerCamelCase    ):
         __SCREAMING_SNAKE_CASE      							=       """""".join(
             self.byte_encoder[b] for b in token.encode("""utf-8"""    )    )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
         bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCamelCase    ).split(""" """    )    )
      return bpe_tokens





   def      UpperCAmelCase__     (					self				:			Optional[Any]				,lowerCamelCase				:			Union[str, Any]    ):

      '''simple docstring'''

      return self.encoder.get(lowerCamelCase				,self.encoder.get(self.unk_token    )    )





   def      UpperCAmelCase__     (					self				:			Any				,lowerCamelCase				:			int    ):

      '''simple docstring'''

      return self.decoder.get(lowerCamelCase    )





   def      UpperCAmelCase__     (					self				:			List[str]				,lowerCamelCase				:			List[Any]    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       """""".join(lowerCamelCase    )
      __SCREAMING_SNAKE_CASE      							=       bytearray([self.byte_decoder[c] for c in text]    ).decode("""utf-8"""				,errors=self.errors    )
      return text





   def      UpperCAmelCase__     (					self				:			Tuple				,lowerCamelCase				:			str				,lowerCamelCase				:			Optional[str] = None    ):

      '''simple docstring'''

      if not os.path.isdir(lowerCamelCase    ):
         logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""    )
         return
      __SCREAMING_SNAKE_CASE      							=       os.path.join(
          lowerCamelCase				,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )
      __SCREAMING_SNAKE_CASE      							=       os.path.join(
          lowerCamelCase				,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]    )

      with open(lowerCamelCase				,"""w"""				,encoding="""utf-8"""    ) as f:
         f.write(json.dumps(self.encoder				,indent=2				,sort_keys=lowerCamelCase				,ensure_ascii=lowerCamelCase    ) + """\n"""    )

      __SCREAMING_SNAKE_CASE      							=       0
      with open(lowerCamelCase				,"""w"""				,encoding="""utf-8"""    ) as writer:
         writer.write("""#version: 0.2\n"""    )
         for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				,key=lambda lowerCamelCase    : kv[1]    ):
            if index != token_index:
               logger.warning(
                   f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                   """ Please check that the tokenizer is not corrupted!"""    )
               __SCREAMING_SNAKE_CASE      							=       token_index
            writer.write(""" """.join(lowerCamelCase    ) + """\n"""    )
            index += 1

      return vocab_file, merge_file





   def      UpperCAmelCase__     (					self				:			Any				,lowerCamelCase				:			List[int]				,lowerCamelCase				:			Optional[List[int]] = None    ):

      '''simple docstring'''

      if token_ids_a is None:
         return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
      __SCREAMING_SNAKE_CASE      							=       [self.cls_token_id]
      __SCREAMING_SNAKE_CASE      							=       [self.sep_token_id]
      return cls + token_ids_a + sep + sep + token_ids_a + sep





   def      UpperCAmelCase__     (					self				:			Tuple				,lowerCamelCase				:			List[int]				,lowerCamelCase				:			Optional[List[int]] = None				,lowerCamelCase				:			bool = False    ):

      '''simple docstring'''

      if already_has_special_tokens:
         return super().get_special_tokens_mask(
             token_ids_a=lowerCamelCase				,token_ids_a=lowerCamelCase				,already_has_special_tokens=lowerCamelCase    )

      if token_ids_a is None:
         return [1] + ([0] * len(lowerCamelCase    )) + [1]
      return [1] + ([0] * len(lowerCamelCase    )) + [1, 1] + ([0] * len(lowerCamelCase    )) + [1]





   def      UpperCAmelCase__     (					self				:			str				,lowerCamelCase				:			List[int]				,lowerCamelCase				:			Optional[List[int]] = None    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       [self.sep_token_id]
      __SCREAMING_SNAKE_CASE      							=       [self.cls_token_id]

      if token_ids_a is None:
         return len(cls + token_ids_a + sep    ) * [0]
      return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]





   def      UpperCAmelCase__     (					self				:			int				,lowerCamelCase				:			Optional[Any]				,lowerCamelCase				:			List[str]=False				,**lowerCamelCase				:			str    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       kwargs.pop("""add_prefix_space"""				,self.add_prefix_space    )
      if (is_split_into_words or add_prefix_space) and (len(lowerCamelCase    ) > 0 and not text[0].isspace()):
         __SCREAMING_SNAKE_CASE      							=       """ """ + text
      return (text, kwargs)












'''simple docstring'''

def 						__magic_name__       (							__UpperCAmelCase      )						->					bool:

   '''simple docstring'''







   if num < 0:
      return False

   __SCREAMING_SNAKE_CASE      							=       num
   __SCREAMING_SNAKE_CASE      							=       0
   while num > 0:
      __SCREAMING_SNAKE_CASE      							=       rev_num * 10 + (num % 10)
      num //= 10

   return num_copy == rev_num


if __name__ == "__main__":
    import doctest

    doctest.testmod()





