





import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowerCAmelCase 				=     logging.get_logger(__name__)


_lowerCAmelCase 				=     {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

_lowerCAmelCase 				=     {
    'vocab_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
    },
}

_lowerCAmelCase 				=     {
    'allenai/longformer-base-4096': 4_096,
    'allenai/longformer-large-4096': 4_096,
    'allenai/longformer-large-4096-finetuned-triviaqa': 4_096,
    'allenai/longformer-base-4096-extra.pos.embd.only': 4_096,
    'allenai/longformer-large-4096-extra.pos.embd.only': 4_096,
}



@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def     _lowerCAmelCase		(  ):
   '''simple docstring'''


   A_     :				Union[str, Any]			      =	(
       list(range(ord("""!"""     )  ,ord("""~"""     ) + 1     )     ) + list(range(ord("""¡"""     )  ,ord("""¬"""     ) + 1     )     ) + list(range(ord("""®"""     )  ,ord("""ÿ"""     ) + 1     )     )
   )
   A_     :				str			      =	bs[:]
   A_     :				Tuple			      =	0
   for b in range(2**8     ):
      if b not in bs:
         bs.append(__UpperCamelCase     )
         cs.append(2**8 + n     )
         n += 1
   A_     :				List[Any]			      =	[chr(__UpperCamelCase     ) for n in cs]
   return dict(zip(__UpperCamelCase  ,__UpperCamelCase     )     )



def     _lowerCAmelCase		(  _lowerCAmelCase     ):
   '''simple docstring'''


   A_     :				Any			      =	set()
   A_     :				Any			      =	word[0]
   for char in word[1:]:
      pairs.add((prev_char, char)     )
      A_     :				int			      =	char
   return pairs







class    _UpperCAmelCase							(		SCREAMING_SNAKE_CASE_     ):

      '''simple docstring'''





      a						       =							VOCAB_FILES_NAMES
      a						       =							PRETRAINED_VOCAB_FILES_MAP
      a						       =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      a						       =							['''input_ids''', '''attention_mask''']
      def __init__(		self      ,       a__      ,       a__      ,       a__="replace"      ,       a__="<s>"      ,       a__="</s>"      ,       a__="</s>"      ,       a__="<s>"      ,       a__="<unk>"      ,       a__="<pad>"      ,       a__="<mask>"      ,       a__=False      ,       **a__      ,       ):
         A_     :				str			      =	AddedToken(UpperCamelCase__      ,       lstrip=UpperCamelCase__      ,       rstrip=UpperCamelCase__       ) if isinstance(UpperCamelCase__      ,       UpperCamelCase__       ) else bos_token
         A_     :				Any			      =	AddedToken(UpperCamelCase__      ,       lstrip=UpperCamelCase__      ,       rstrip=UpperCamelCase__       ) if isinstance(UpperCamelCase__      ,       UpperCamelCase__       ) else eos_token
         A_     :				List[str]			      =	AddedToken(UpperCamelCase__      ,       lstrip=UpperCamelCase__      ,       rstrip=UpperCamelCase__       ) if isinstance(UpperCamelCase__      ,       UpperCamelCase__       ) else sep_token
         A_     :				Union[str, Any]			      =	AddedToken(UpperCamelCase__      ,       lstrip=UpperCamelCase__      ,       rstrip=UpperCamelCase__       ) if isinstance(UpperCamelCase__      ,       UpperCamelCase__       ) else cls_token
         A_     :				Dict			      =	AddedToken(UpperCamelCase__      ,       lstrip=UpperCamelCase__      ,       rstrip=UpperCamelCase__       ) if isinstance(UpperCamelCase__      ,       UpperCamelCase__       ) else unk_token
         A_     :				Optional[Any]			      =	AddedToken(UpperCamelCase__      ,       lstrip=UpperCamelCase__      ,       rstrip=UpperCamelCase__       ) if isinstance(UpperCamelCase__      ,       UpperCamelCase__       ) else pad_token

         # Mask token behave like a normal word, i.e. include the space before it
         A_     :				List[Any]			      =	AddedToken(UpperCamelCase__      ,       lstrip=UpperCamelCase__      ,       rstrip=UpperCamelCase__       ) if isinstance(UpperCamelCase__      ,       UpperCamelCase__       ) else mask_token

         super().__init__(
             errors=UpperCamelCase__      ,       bos_token=UpperCamelCase__      ,       eos_token=UpperCamelCase__      ,       unk_token=UpperCamelCase__      ,       sep_token=UpperCamelCase__      ,       cls_token=UpperCamelCase__      ,       pad_token=UpperCamelCase__      ,       mask_token=UpperCamelCase__      ,       add_prefix_space=UpperCamelCase__      ,       **UpperCamelCase__      ,       )

         with open(UpperCamelCase__      ,       encoding="""utf-8"""       ) as vocab_handle:
            A_     :				Optional[int]			      =	json.load(UpperCamelCase__       )
         A_     :				Optional[Any]			      =	{v: k for k, v in self.encoder.items()}
         A_     :				int			      =	errors  # how to handle errors in decoding
         A_     :				List[Any]			      =	bytes_to_unicode()
         A_     :				List[Any]			      =	{v: k for k, v in self.byte_encoder.items()}
         with open(UpperCamelCase__      ,       encoding="""utf-8"""       ) as merges_handle:
            A_     :				Union[str, Any]			      =	merges_handle.read().split("""\n"""       )[1:-1]
         A_     :				Tuple			      =	[tuple(merge.split()       ) for merge in bpe_merges]
         A_     :				Optional[int]			      =	dict(zip(UpperCamelCase__      ,       range(len(UpperCamelCase__       )       )       )       )
         A_     :				List[Any]			      =	{}
         A_     :				List[Any]			      =	add_prefix_space

         # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
         A_     :				Dict			      =	re.compile(R"""\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""       )
      @property
      def 		_lowerCamelCase      (		self       ):
         return len(self.encoder       )
      def 		_lowerCamelCase      (		self       ):
         return dict(self.encoder      ,       **self.added_tokens_encoder       )
      def 		_lowerCamelCase      (		self      ,       a__       ):
         if token in self.cache:
            return self.cache[token]
         A_     :				List[str]			      =	tuple(UpperCamelCase__       )
         A_     :				Union[str, Any]			      =	get_pairs(UpperCamelCase__       )

         if not pairs:
            return token

         while True:
            A_     :				List[Any]			      =	min(UpperCamelCase__      ,       key=lambda a__       : self.bpe_ranks.get(UpperCamelCase__      ,       float("""inf"""       )       )       )
            if bigram not in self.bpe_ranks:
               break
            A_     :				str			      =	bigram
            A_     :				str			      =	[]
            A_     :				List[str]			      =	0
            while i < len(UpperCamelCase__       ):
               try:
                  A_     :				List[str]			      =	word.index(UpperCamelCase__      ,       UpperCamelCase__       )
               except ValueError:
                  new_word.extend(word[i:]       )
                  break
               else:
                  new_word.extend(word[i:j]       )
                  A_     :				Optional[int]			      =	j

               if word[i] == first and i < len(UpperCamelCase__       ) - 1 and word[i + 1] == second:
                  new_word.append(first + second       )
                  i += 2
               else:
                  new_word.append(word[i]       )
                  i += 1
            A_     :				List[str]			      =	tuple(UpperCamelCase__       )
            A_     :				Union[str, Any]			      =	new_word
            if len(UpperCamelCase__       ) == 1:
               break
            else:
               A_     :				Union[str, Any]			      =	get_pairs(UpperCamelCase__       )
         A_     :				Optional[int]			      =	''' '''.join(UpperCamelCase__       )
         A_     :				Any			      =	word
         return word
      def 		_lowerCamelCase      (		self      ,       a__       ):
         A_     :				str			      =	[]
         for token in re.findall(self.pat      ,       UpperCamelCase__       ):
            A_     :				int			      =	''''''.join(
                self.byte_encoder[b] for b in token.encode("""utf-8"""       )       )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
            bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCamelCase__       ).split(""" """       )       )
         return bpe_tokens
      def 		_lowerCamelCase      (		self      ,       a__       ):
         return self.encoder.get(UpperCamelCase__      ,       self.encoder.get(self.unk_token       )       )
      def 		_lowerCamelCase      (		self      ,       a__       ):
         return self.decoder.get(UpperCamelCase__       )
      def 		_lowerCamelCase      (		self      ,       a__       ):
         A_     :				str			      =	''''''.join(UpperCamelCase__       )
         A_     :				int			      =	bytearray([self.byte_decoder[c] for c in text]       ).decode("""utf-8"""      ,       errors=self.errors       )
         return text
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__ = None       ):
         if not os.path.isdir(UpperCamelCase__       ):
            logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""       )
            return
         A_     :				str			      =	os.path.join(
             UpperCamelCase__      ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]       )
         A_     :				Tuple			      =	os.path.join(
             UpperCamelCase__      ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]       )

         with open(UpperCamelCase__      ,       """w"""      ,       encoding="""utf-8"""       ) as f:
            f.write(json.dumps(self.encoder      ,       indent=2      ,       sort_keys=UpperCamelCase__      ,       ensure_ascii=UpperCamelCase__       ) + """\n"""       )

         A_     :				Tuple			      =	0
         with open(UpperCamelCase__      ,       """w"""      ,       encoding="""utf-8"""       ) as writer:
            writer.write("""#version: 0.2\n"""       )
            for bpe_tokens, token_index in sorted(self.bpe_ranks.items()      ,       key=lambda a__       : kv[1]       ):
               if index != token_index:
                  logger.warning(
                      F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                      """ Please check that the tokenizer is not corrupted!"""       )
                  A_     :				Optional[int]			      =	token_index
               writer.write(""" """.join(UpperCamelCase__       ) + """\n"""       )
               index += 1

         return vocab_file, merge_file
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__ = None       ):
         if token_ids_a is None:
            return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         A_     :				List[str]			      =	[self.cls_token_id]
         A_     :				List[Any]			      =	[self.sep_token_id]
         return cls + token_ids_a + sep + sep + token_ids_a + sep
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__ = None      ,       a__ = False       ):
         if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_a=UpperCamelCase__      ,       token_ids_a=UpperCamelCase__      ,       already_has_special_tokens=UpperCamelCase__       )

         if token_ids_a is None:
            return [1] + ([0] * len(UpperCamelCase__       )) + [1]
         return [1] + ([0] * len(UpperCamelCase__       )) + [1, 1] + ([0] * len(UpperCamelCase__       )) + [1]
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__ = None       ):
         A_     :				Any			      =	[self.sep_token_id]
         A_     :				List[str]			      =	[self.cls_token_id]

         if token_ids_a is None:
            return len(cls + token_ids_a + sep       ) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]






      def 		_lowerCamelCase      (		self      ,       a__      ,       a__=False      ,       **a__       ):
         A_     :				int			      =	kwargs.pop("""add_prefix_space"""      ,       self.add_prefix_space       )
         if (is_split_into_words or add_prefix_space) and (len(UpperCamelCase__       ) > 0 and not text[0].isspace()):
            A_     :				List[Any]			      =	''' ''' + text
         return (text, kwargs)













from typing import List

from .keymap import KEYMAP, get_character



def     _lowerCAmelCase		(  _lowerCAmelCase     ):
   '''simple docstring'''


   def decorator(_lowerCAmelCase     ):
      A_     :				List[Any]			      =	getattr(_lowerCAmelCase  ,"""handle_key"""  ,[]     )
      handle += [key]
      setattr(_lowerCAmelCase  ,"""handle_key"""  ,_lowerCAmelCase     )
      return func

   return decorator



def     _lowerCAmelCase		(  *_lowerCAmelCase     ):
   '''simple docstring'''


   def decorator(_lowerCAmelCase     ):
      A_     :				Tuple			      =	getattr(_lowerCAmelCase  ,"""handle_key"""  ,[]     )
      handle += keys
      setattr(_lowerCAmelCase  ,"""handle_key"""  ,_lowerCAmelCase     )
      return func

   return decorator

class    _UpperCAmelCase							(		_lowerCamelCase     ):
      def __new__(		cls      ,       a__      ,       a__      ,       a__       ):
         A_     :				Any			      =	super().__new__(cls      ,       a__      ,       a__      ,       a__       )
         if not hasattr(a__      ,       """key_handler"""       ):
            setattr(a__      ,       """key_handler"""      ,       {}       )
         setattr(a__      ,       """handle_input"""      ,       KeyHandler.handle_input       )

         for value in attrs.values():
            A_     :				Tuple			      =	getattr(a__      ,       """handle_key"""      ,       []       )
            for key in handled_keys:
               A_     :				Optional[Any]			      =	value
         return new_cls






      @staticmethod
      def 		_lowerCamelCase      (		cls       ):
         A_     :				List[str]			      =	get_character()
         if char != KEYMAP["undefined"]:
            A_     :				str			      =	ord(a__       )
         A_     :				List[str]			      =	cls.key_handler.get(a__       )
         if handler:
            A_     :				Optional[int]			      =	char
            return handler(cls       )
         else:
            return None






def     _lowerCAmelCase		(  cls     ):
   '''simple docstring'''


   return KeyHandler(cls.__name__  ,cls.__bases__  ,cls.__dict__.copy()     )






