

import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__UpperCAmelCase    				=     logging.get_logger(__name__)


__UpperCAmelCase    				=     {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

__UpperCAmelCase    				=     {
    'vocab_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
    },
}

__UpperCAmelCase    				=     {
    'allenai/longformer-base-4096': 4_0_9_6,
    'allenai/longformer-large-4096': 4_0_9_6,
    'allenai/longformer-large-4096-finetuned-triviaqa': 4_0_9_6,
    'allenai/longformer-base-4096-extra.pos.embd.only': 4_0_9_6,
    'allenai/longformer-large-4096-extra.pos.embd.only': 4_0_9_6,
}







@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def 	_snake_case		(    )       ->      Optional[Any]:

     """simple docstring"""







     _lowerCAmelCase   :					List[Any]        =    (
         list(range(ord("!"				)		,							ord("~"				) + 1				)				) + list(range(ord("¡"				)		,							ord("¬"				) + 1				)				) + list(range(ord("®"				)		,							ord("ÿ"				) + 1				)				)
     )
     _lowerCAmelCase   :					Any        =    bs[:]
     _lowerCAmelCase   :					List[str]        =    0
     for b in range(2**8				):
          if b not in bs:
               bs.append(SCREAMING_SNAKE_CASE				)
               cs.append(2**8 + n				)
               n += 1
     _lowerCAmelCase   :					Optional[Any]        =    [chr(SCREAMING_SNAKE_CASE				) for n in cs]
     return dict(zip(SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE				)				)







def 	_snake_case		(    SCREAMING_SNAKE_CASE				)       ->      Optional[Any]:

     """simple docstring"""







     _lowerCAmelCase   :					str        =    set()
     _lowerCAmelCase   :					List[Any]        =    word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)				)
          _lowerCAmelCase   :					Any        =    char
     return pairs





class UpperCAmelCase_    ( A ):



 """simple docstring"""







 _lowercase     :					Union[str, Any]      =						VOCAB_FILES_NAMES
 _lowercase     :					Tuple      =						PRETRAINED_VOCAB_FILES_MAP
 _lowercase     :					Dict      =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _lowercase     :					str      =						['''input_ids''', '''attention_mask''']

 def __init__(		self :	Tuple     ,						A_ :	Any     ,						A_ :	List[str]     ,						A_ :	Optional[int]="replace"     ,						A_ :	Any="<s>"     ,						A_ :	List[Any]="</s>"     ,						A_ :	List[Any]="</s>"     ,						A_ :	Optional[Any]="<s>"     ,						A_ :	Union[str, Any]="<unk>"     ,						A_ :	Dict="<pad>"     ,						A_ :	Any="<mask>"     ,						A_ :	int=False     ,						**A_ :	Tuple     ,						):






      '''simple docstring'''




      _lowerCAmelCase   :					List[Any]        =    AddedToken(A_     ,						lstrip=A_     ,						rstrip=A_       ) if isinstance(A_     ,						A_       ) else bos_token
      _lowerCAmelCase   :					Union[str, Any]        =    AddedToken(A_     ,						lstrip=A_     ,						rstrip=A_       ) if isinstance(A_     ,						A_       ) else eos_token
      _lowerCAmelCase   :					Optional[Any]        =    AddedToken(A_     ,						lstrip=A_     ,						rstrip=A_       ) if isinstance(A_     ,						A_       ) else sep_token
      _lowerCAmelCase   :					int        =    AddedToken(A_     ,						lstrip=A_     ,						rstrip=A_       ) if isinstance(A_     ,						A_       ) else cls_token
      _lowerCAmelCase   :					Dict        =    AddedToken(A_     ,						lstrip=A_     ,						rstrip=A_       ) if isinstance(A_     ,						A_       ) else unk_token
      _lowerCAmelCase   :					Tuple        =    AddedToken(A_     ,						lstrip=A_     ,						rstrip=A_       ) if isinstance(A_     ,						A_       ) else pad_token

      # Mask token behave like a normal word, i.e. include the space before it
      _lowerCAmelCase   :					Optional[int]        =    AddedToken(A_     ,						lstrip=A_     ,						rstrip=A_       ) if isinstance(A_     ,						A_       ) else mask_token

      super().__init__(
          errors=A_     ,						bos_token=A_     ,						eos_token=A_     ,						unk_token=A_     ,						sep_token=A_     ,						cls_token=A_     ,						pad_token=A_     ,						mask_token=A_     ,						add_prefix_space=A_     ,						**A_     ,						)

      with open(A_     ,						encoding="utf-8"       ) as vocab_handle:
           _lowerCAmelCase   :					Union[str, Any]        =    json.load(A_       )
      _lowerCAmelCase   :					List[Any]        =    {v: k for k, v in self.encoder.items()}
      _lowerCAmelCase   :					Any        =    errors  # how to handle errors in decoding
      _lowerCAmelCase   :					Any        =    bytes_to_unicode()
      _lowerCAmelCase   :					Optional[int]        =    {v: k for k, v in self.byte_encoder.items()}
      with open(A_     ,						encoding="utf-8"       ) as merges_handle:
           _lowerCAmelCase   :					int        =    merges_handle.read().split("\n"       )[1:-1]
      _lowerCAmelCase   :					Optional[int]        =    [tuple(merge.split()       ) for merge in bpe_merges]
      _lowerCAmelCase   :					Tuple        =    dict(zip(A_     ,						range(len(A_       )       )       )       )
      _lowerCAmelCase   :					Union[str, Any]        =    {}
      _lowerCAmelCase   :					List[str]        =    add_prefix_space

      # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
      _lowerCAmelCase   :					List[str]        =    re.compile(r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"       )

 @property
 def    __magic_name__       (		self :	str       ):






      '''simple docstring'''




      return len(self.encoder       )

 def    __magic_name__       (		self :	Optional[Any]       ):






      '''simple docstring'''




      return dict(self.encoder     ,						**self.added_tokens_encoder       )

 def    __magic_name__       (		self :	Optional[int]     ,						A_ :	List[str]       ):






      '''simple docstring'''




      if token in self.cache:
           return self.cache[token]
      _lowerCAmelCase   :					List[str]        =    tuple(A_       )
      _lowerCAmelCase   :					Dict        =    get_pairs(A_       )

      if not pairs:
           return token

      while True:
           _lowerCAmelCase   :					Tuple        =    min(A_     ,						key=lambda A_       : self.bpe_ranks.get(A_     ,						float("inf"       )       )       )
           if bigram not in self.bpe_ranks:
                break
           _lowerCAmelCase   :					Optional[int]        =    bigram
           _lowerCAmelCase   :					Tuple        =    []
           _lowerCAmelCase   :					Dict        =    0
           while i < len(A_       ):
                try:
                     _lowerCAmelCase   :					int        =    word.index(A_     ,						A_       )
                except ValueError:
                     new_word.extend(word[i:]       )
                     break
                else:
                     new_word.extend(word[i:j]       )
                     _lowerCAmelCase   :					str        =    j

                if word[i] == first and i < len(A_       ) - 1 and word[i + 1] == second:
                     new_word.append(first + second       )
                     i += 2
                else:
                     new_word.append(word[i]       )
                     i += 1
           _lowerCAmelCase   :					List[Any]        =    tuple(A_       )
           _lowerCAmelCase   :					Dict        =    new_word
           if len(A_       ) == 1:
                break
           else:
                _lowerCAmelCase   :					Optional[Any]        =    get_pairs(A_       )
      _lowerCAmelCase   :					Dict        =    " ".join(A_       )
      _lowerCAmelCase   :					Any        =    word
      return word

 def    __magic_name__       (		self :	int     ,						A_ :	int       ):






      '''simple docstring'''




      _lowerCAmelCase   :					Union[str, Any]        =    []
      for token in re.findall(self.pat     ,						A_       ):
           _lowerCAmelCase   :					str        =    "".join(
               self.byte_encoder[b] for b in token.encode("utf-8"       )       )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
           bpe_tokens.extend(bpe_token for bpe_token in self.bpe(A_       ).split(" "       )       )
      return bpe_tokens

 def    __magic_name__       (		self :	str     ,						A_ :	Dict       ):






      '''simple docstring'''




      return self.encoder.get(A_     ,						self.encoder.get(self.unk_token       )       )

 def    __magic_name__       (		self :	Optional[Any]     ,						A_ :	str       ):






      '''simple docstring'''




      return self.decoder.get(A_       )

 def    __magic_name__       (		self :	Optional[int]     ,						A_ :	List[Any]       ):






      '''simple docstring'''




      _lowerCAmelCase   :					Any        =    "".join(A_       )
      _lowerCAmelCase   :					Tuple        =    bytearray([self.byte_decoder[c] for c in text]       ).decode("utf-8"     ,						errors=self.errors       )
      return text

 def    __magic_name__       (		self :	List[str]     ,						A_ :	str     ,						A_ :	Optional[str] = None       ):






      '''simple docstring'''




      if not os.path.isdir(A_       ):
           logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''       )
           return
      _lowerCAmelCase   :					List[Any]        =    os.path.join(
          A_     ,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]       )
      _lowerCAmelCase   :					Optional[int]        =    os.path.join(
          A_     ,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]       )

      with open(A_     ,						"w"     ,						encoding="utf-8"       ) as f:
           f.write(json.dumps(self.encoder     ,						indent=2     ,						sort_keys=A_     ,						ensure_ascii=A_       ) + "\n"       )

      _lowerCAmelCase   :					List[str]        =    0
      with open(A_     ,						"w"     ,						encoding="utf-8"       ) as writer:
           writer.write("#version: 0.2\n"       )
           for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     ,						key=lambda A_       : kv[1]       ):
                if index != token_index:
                     logger.warning(
                         F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                         " Please check that the tokenizer is not corrupted!"       )
                     _lowerCAmelCase   :					List[str]        =    token_index
                writer.write(" ".join(A_       ) + "\n"       )
                index += 1

      return vocab_file, merge_file

 def    __magic_name__       (		self :	Any     ,						A_ :	List[int]     ,						A_ :	Optional[List[int]] = None       ):






      '''simple docstring'''




      if token_ids_a is None:
           return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
      _lowerCAmelCase   :					int        =    [self.cls_token_id]
      _lowerCAmelCase   :					List[str]        =    [self.sep_token_id]
      return cls + token_ids_a + sep + sep + token_ids_a + sep

 def    __magic_name__       (		self :	Any     ,						A_ :	List[int]     ,						A_ :	Optional[List[int]] = None     ,						A_ :	bool = False       ):






      '''simple docstring'''




      if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=A_     ,						token_ids_a=A_     ,						already_has_special_tokens=A_       )

      if token_ids_a is None:
           return [1] + ([0] * len(A_       )) + [1]
      return [1] + ([0] * len(A_       )) + [1, 1] + ([0] * len(A_       )) + [1]

 def    __magic_name__       (		self :	str     ,						A_ :	List[int]     ,						A_ :	Optional[List[int]] = None       ):






      '''simple docstring'''




      _lowerCAmelCase   :					List[Any]        =    [self.sep_token_id]
      _lowerCAmelCase   :					Any        =    [self.cls_token_id]

      if token_ids_a is None:
           return len(cls + token_ids_a + sep       ) * [0]
      return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]


 def    __magic_name__       (		self :	Union[str, Any]     ,						A_ :	int     ,						A_ :	Tuple=False     ,						**A_ :	Union[str, Any]       ):






      '''simple docstring'''




      _lowerCAmelCase   :					Any        =    kwargs.pop("add_prefix_space"     ,						self.add_prefix_space       )
      if (is_split_into_words or add_prefix_space) and (len(A_       ) > 0 and not text[0].isspace()):
           _lowerCAmelCase   :					Optional[Any]        =    " " + text
      return (text, kwargs)









def 	_snake_case		(    SCREAMING_SNAKE_CASE				)       ->      "list[int]":

     """simple docstring"""







     if upper_limit < 0:
          raise ValueError("Limit for the Catalan sequence must be ≥ 0"				)

     _lowerCAmelCase   :					Optional[Any]        =    [0] * (upper_limit + 1)

     # Base case: C(0) = C(1) = 1
     _lowerCAmelCase   :					Optional[int]        =    1
     if upper_limit > 0:
          _lowerCAmelCase   :					Any        =    1

     # Recurrence relation: C(i) = sum(C(j).C(i-j-1)), from j = 0 to i
     for i in range(2		,							upper_limit + 1				):
          for j in range(SCREAMING_SNAKE_CASE				):
               catalan_list[i] += catalan_list[j] * catalan_list[i - j - 1]

     return catalan_list


if __name__ == "__main__":
  print('\n********* Catalan Numbers Using Dynamic Programming ************\n')
  print('\n*** Enter -1 at any time to quit ***')
  print('\nEnter the upper limit (≥ 0) for the Catalan number sequence: ', end='')
  try:
    while True:
      __UpperCAmelCase    				=     int(input().strip())
      if N < 0:
        print('\n********* Goodbye!! ************')
        break
      else:
        print(f'''The Catalan numbers from 0 through {N} are:''')
        print(catalan_numbers(N))
        print('Try another upper limit for the sequence: ', end='')
  except (NameError, ValueError):
    print('\n********* Invalid input, goodbye! ************\n')

  import doctest

  doctest.testmod()






