
'''simple docstring'''
from ...configuration_utils import PretrainedConfig
from ...utils import logging


A__   :		Union[str, Any] 		=logging.get_logger(__name__)

A__   :		Union[str, Any] 		={
    'transfo-xl-wt103': 'https://huggingface.co/transfo-xl-wt103/resolve/main/config.json',
}







class  __A				( _SCREAMING_SNAKE_CASE			):
 lowerCamelCase				       ='''transfo-xl'''
 lowerCamelCase				       =['''mems''']
 lowerCamelCase				       ={
     '''n_token''': '''vocab_size''',
     '''hidden_size''': '''d_model''',
     '''num_attention_heads''': '''n_head''',
     '''num_hidden_layers''': '''n_layer''',
 }
 def __init__(   self			:    Dict				,   lowerCamelCase			:    int=26_77_35				,   lowerCamelCase			:    List[str]=[2_00_00, 4_00_00, 20_00_00]				,   lowerCamelCase			:    Tuple=10_24				,   lowerCamelCase			:    Union[str, Any]=10_24				,   lowerCamelCase			:    Any=16				,   lowerCamelCase			:    str=64				,   lowerCamelCase			:    str=40_96				,   lowerCamelCase			:    Dict=4				,   lowerCamelCase			:    List[str]=False				,   lowerCamelCase			:    Optional[Any]=18				,   lowerCamelCase			:    Any=16_00				,   lowerCamelCase			:    Dict=10_00				,   lowerCamelCase			:    Dict=True				,   lowerCamelCase			:    Union[str, Any]=True				,   lowerCamelCase			:    str=0				,   lowerCamelCase			:    Union[str, Any]=-1				,   lowerCamelCase			:    Optional[int]=True				,   lowerCamelCase			:    Optional[Any]=0.1				,   lowerCamelCase			:    Tuple=0.0				,   lowerCamelCase			:    Optional[Any]=True				,   lowerCamelCase			:    List[Any]="normal"				,   lowerCamelCase			:    Optional[int]=0.01				,   lowerCamelCase			:    Dict=0.01				,   lowerCamelCase			:    int=0.02				,   lowerCamelCase			:    Dict=1e-5				,   lowerCamelCase			:    str=0				,   **lowerCamelCase			:    Any				,   ):



   """simple docstring"""





   __A		:     Any												=		vocab_size
   __A		:     Optional[int]												=		[]
   self.cutoffs.extend(lowerCamelCase       )
   if proj_share_all_but_first:
     __A		:     List[Any]												=		[False] + [True] * len(self.cutoffs       )
   else:
     __A		:     List[Any]												=		[False] + [False] * len(self.cutoffs       )
   __A		:     Union[str, Any]												=		d_model
   __A		:     Optional[Any]												=		d_embed
   __A		:     int												=		d_head
   __A		:     int												=		d_inner
   __A		:     Any												=		div_val
   __A		:     Optional[Any]												=		pre_lnorm
   __A		:     int												=		n_layer
   __A		:     Optional[int]												=		n_head
   __A		:     Optional[int]												=		mem_len
   __A		:     List[str]												=		same_length
   __A		:     int												=		attn_type
   __A		:     Any												=		clamp_len
   __A		:     Optional[Any]												=		sample_softmax
   __A		:     Tuple												=		adaptive
   __A		:     List[str]												=		dropout
   __A		:     str												=		dropatt
   __A		:     Optional[Any]												=		untie_r
   __A		:     Tuple												=		init
   __A		:     str												=		init_range
   __A		:     Any												=		proj_init_std
   __A		:     Any												=		init_std
   __A		:     int												=		layer_norm_epsilon
   super().__init__(eos_token_id=lowerCamelCase				,   **lowerCamelCase       )
 @property
 def 		lowercase_(   self			:    Dict       ):



   """simple docstring"""





   logger.info(f"The model {self.model_type} is one of the few models that has no sequence length limit."       )
   return -1






 @max_position_embeddings.setter
 def 		lowercase_(   self			:    List[Any]				,   lowerCamelCase			:    Optional[Any]       ):



   """simple docstring"""





   raise NotImplementedError(
       f"The model {self.model_type} is one of the few models that has no sequence length limit."       )





'''simple docstring'''
import json
import os
from functools import lru_cache
from typing import Dict, List, Optional, Tuple, Union

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...tokenization_utils_base import BatchEncoding, EncodedInput
from ...utils import PaddingStrategy, logging


A__   :		Union[str, Any] 		=logging.get_logger(__name__)


A__   :		List[str] 		={'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

# See all LED models at https://huggingface.co/models?filter=LED
A__   :		str 		={
    'vocab_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json',
    },
    'merges_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json',
    },
}

A__   :		List[Any] 		={
    'allenai/led-base-16384': 16_384,
}







@lru_cache()
# Copied from transformers.models.bart.tokenization_bart.bytes_to_unicode
def        A_     (						)				->       Tuple:





  """simple docstring"""

  __A		:     Optional[int]												=		(
      list(range(ord("""!""" )			,     ord("""~""" ) + 1 ) ) + list(range(ord("""¡""" )			,     ord("""¬""" ) + 1 ) ) + list(range(ord("""®""" )			,     ord("""ÿ""" ) + 1 ) )
  )
  __A		:     List[Any]												=		bs[:]
  __A		:     Union[str, Any]												=		0
  for b in range(2**8 ):
    if b not in bs:
      bs.append(__SCREAMING_SNAKE_CASE )
      cs.append(2**8 + n )
      n += 1
  __A		:     int												=		[chr(__SCREAMING_SNAKE_CASE ) for n in cs]
  return dict(zip(__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE ) )







def        A_     (						__SCREAMING_SNAKE_CASE						:  str )				->       Tuple:





  """simple docstring"""

  __A		:     str												=		set()
  __A		:     int												=		word[0]
  for char in word[1:]:
    pairs.add((prev_char, char) )
    __A		:     Optional[Any]												=		char
  return pairs







class  __A				( _SCREAMING_SNAKE_CASE			):
 lowerCamelCase				       =VOCAB_FILES_NAMES
 lowerCamelCase				       =PRETRAINED_VOCAB_FILES_MAP
 lowerCamelCase				       =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowerCamelCase				       =['''input_ids''', '''attention_mask''']
 def __init__(   self			:    Any				,   lowerCamelCase			:    Dict				,   lowerCamelCase			:    List[str]				,   lowerCamelCase			:    Dict="replace"				,   lowerCamelCase			:    int="<s>"				,   lowerCamelCase			:    str="</s>"				,   lowerCamelCase			:    List[str]="</s>"				,   lowerCamelCase			:    Optional[Any]="<s>"				,   lowerCamelCase			:    Any="<unk>"				,   lowerCamelCase			:    str="<pad>"				,   lowerCamelCase			:    Any="<mask>"				,   lowerCamelCase			:    Optional[int]=False				,   **lowerCamelCase			:    str				,   ):



   """simple docstring"""





   __A		:     List[Any]												=		AddedToken(lowerCamelCase				,   lstrip=lowerCamelCase				,   rstrip=lowerCamelCase       ) if isinstance(lowerCamelCase				,   lowerCamelCase       ) else bos_token
   __A		:     Any												=		AddedToken(lowerCamelCase				,   lstrip=lowerCamelCase				,   rstrip=lowerCamelCase       ) if isinstance(lowerCamelCase				,   lowerCamelCase       ) else eos_token
   __A		:     int												=		AddedToken(lowerCamelCase				,   lstrip=lowerCamelCase				,   rstrip=lowerCamelCase       ) if isinstance(lowerCamelCase				,   lowerCamelCase       ) else sep_token
   __A		:     Any												=		AddedToken(lowerCamelCase				,   lstrip=lowerCamelCase				,   rstrip=lowerCamelCase       ) if isinstance(lowerCamelCase				,   lowerCamelCase       ) else cls_token
   __A		:     List[Any]												=		AddedToken(lowerCamelCase				,   lstrip=lowerCamelCase				,   rstrip=lowerCamelCase       ) if isinstance(lowerCamelCase				,   lowerCamelCase       ) else unk_token
   __A		:     List[Any]												=		AddedToken(lowerCamelCase				,   lstrip=lowerCamelCase				,   rstrip=lowerCamelCase       ) if isinstance(lowerCamelCase				,   lowerCamelCase       ) else pad_token

   # Mask token behave like a normal word, i.e. include the space before it
   __A		:     List[str]												=		AddedToken(lowerCamelCase				,   lstrip=lowerCamelCase				,   rstrip=lowerCamelCase       ) if isinstance(lowerCamelCase				,   lowerCamelCase       ) else mask_token

   super().__init__(
       errors=lowerCamelCase				,   bos_token=lowerCamelCase				,   eos_token=lowerCamelCase				,   unk_token=lowerCamelCase				,   sep_token=lowerCamelCase				,   cls_token=lowerCamelCase				,   pad_token=lowerCamelCase				,   mask_token=lowerCamelCase				,   add_prefix_space=lowerCamelCase				,   **lowerCamelCase				,   )

   with open(lowerCamelCase				,   encoding="""utf-8"""       ) as vocab_handle:
     __A		:     Optional[Any]												=		json.load(lowerCamelCase       )
   __A		:     Dict												=		{v: k for k, v in self.encoder.items()}
   __A		:     List[Any]												=		errors  # how to handle errors in decoding
   __A		:     Optional[int]												=		bytes_to_unicode()
   __A		:     str												=		{v: k for k, v in self.byte_encoder.items()}
   with open(lowerCamelCase				,   encoding="""utf-8"""       ) as merges_handle:
     __A		:     str												=		merges_handle.read().split("""\n"""       )[1:-1]
   __A		:     Tuple												=		[tuple(merge.split()       ) for merge in bpe_merges]
   __A		:     List[Any]												=		dict(zip(lowerCamelCase				,   range(len(lowerCamelCase       )       )       )       )
   __A		:     List[str]												=		{}
   __A		:     List[str]												=		add_prefix_space

   # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
   __A		:     Any												=		re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""       )
 @property
 # Copied from transformers.models.bart.tokenization_bart.BartTokenizer.vocab_size
 def 		lowercase_(   self			:    Any       ):



   """simple docstring"""





   return len(self.encoder       )
 def 		lowercase_(   self			:    Optional[int]       ):



   """simple docstring"""





   return dict(self.encoder				,   **self.added_tokens_encoder       )
 def 		lowercase_(   self			:    str				,   lowerCamelCase			:    Optional[int]       ):



   """simple docstring"""





   if token in self.cache:
     return self.cache[token]
   __A		:     Optional[Any]												=		tuple(lowerCamelCase       )
   __A		:     Dict												=		get_pairs(lowerCamelCase       )

   if not pairs:
     return token

   while True:
     __A		:     str												=		min(lowerCamelCase				,   key=lambda lowerCamelCase       : self.bpe_ranks.get(lowerCamelCase				,   float("""inf"""       )       )       )
     if bigram not in self.bpe_ranks:
       break
     __A							,					__A		:     Optional[Any]												=		bigram
     __A		:     List[Any]												=		[]
     __A		:     Union[str, Any]												=		0
     while i < len(lowerCamelCase       ):
       try:
         __A		:     str												=		word.index(lowerCamelCase				,   lowerCamelCase       )
       except ValueError:
         new_word.extend(word[i:]       )
         break
       else:
         new_word.extend(word[i:j]       )
         __A		:     Union[str, Any]												=		j

       if word[i] == first and i < len(lowerCamelCase       ) - 1 and word[i + 1] == second:
         new_word.append(first + second       )
         i += 2
       else:
         new_word.append(word[i]       )
         i += 1
     __A		:     Dict												=		tuple(lowerCamelCase       )
     __A		:     List[Any]												=		new_word
     if len(lowerCamelCase       ) == 1:
       break
     else:
       __A		:     Optional[Any]												=		get_pairs(lowerCamelCase       )
   __A		:     Optional[int]												=		""" """.join(lowerCamelCase       )
   __A		:     str												=		word
   return word
 def 		lowercase_(   self			:    str				,   lowerCamelCase			:    Optional[int]       ):



   """simple docstring"""





   __A		:     List[str]												=		[]
   for token in re.findall(self.pat				,   lowerCamelCase       ):
     __A		:     Optional[Any]												=		"""""".join(
         self.byte_encoder[b] for b in token.encode("""utf-8"""       )       )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
     bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCamelCase       ).split(""" """       )       )
   return bpe_tokens
 def 		lowercase_(   self			:    Dict				,   lowerCamelCase			:    Tuple       ):



   """simple docstring"""





   return self.encoder.get(lowerCamelCase				,   self.encoder.get(self.unk_token       )       )
 def 		lowercase_(   self			:    str				,   lowerCamelCase			:    Optional[Any]       ):



   """simple docstring"""





   return self.decoder.get(lowerCamelCase       )
 def 		lowercase_(   self			:    Tuple				,   lowerCamelCase			:    int       ):



   """simple docstring"""





   __A		:     Any												=		"""""".join(lowerCamelCase       )
   __A		:     int												=		bytearray([self.byte_decoder[c] for c in text]       ).decode("""utf-8"""				,   errors=self.errors       )
   return text
 def 		lowercase_(   self			:    List[str]				,   lowerCamelCase			:    str				,   lowerCamelCase			:    Optional[str] = None       ):



   """simple docstring"""





   if not os.path.isdir(lowerCamelCase       ):
     logger.error(f"Vocabulary path ({save_directory}) should be a directory"       )
     return
   __A		:     Any												=		os.path.join(
       lowerCamelCase				,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]       )
   __A		:     Optional[Any]												=		os.path.join(
       lowerCamelCase				,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]       )

   with open(lowerCamelCase				,   """w"""				,   encoding="""utf-8"""       ) as f:
     f.write(json.dumps(self.encoder				,   indent=2				,   sort_keys=lowerCamelCase				,   ensure_ascii=lowerCamelCase       ) + """\n"""       )

   __A		:     str												=		0
   with open(lowerCamelCase				,   """w"""				,   encoding="""utf-8"""       ) as writer:
     writer.write("""#version: 0.2\n"""       )
     for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				,   key=lambda lowerCamelCase       : kv[1]       ):
       if index != token_index:
         logger.warning(
             f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
             """ Please check that the tokenizer is not corrupted!"""       )
         __A		:     Any												=		token_index
       writer.write(""" """.join(lowerCamelCase       ) + """\n"""       )
       index += 1

   return vocab_file, merge_file
 def 		lowercase_(   self			:    str				,   lowerCamelCase			:    List[int]				,   lowerCamelCase			:    Optional[List[int]] = None       ):



   """simple docstring"""





   if token_ids_a is None:
     return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
   __A		:     Tuple												=		[self.cls_token_id]
   __A		:     str												=		[self.sep_token_id]
   return cls + token_ids_a + sep + sep + token_ids_a + sep
 def 		lowercase_(   self			:    List[str]				,   lowerCamelCase			:    List[int]				,   lowerCamelCase			:    Optional[List[int]] = None				,   lowerCamelCase			:    bool = False       ):



   """simple docstring"""





   if already_has_special_tokens:
     return super().get_special_tokens_mask(
         token_ids_a=lowerCamelCase				,   token_ids_a=lowerCamelCase				,   already_has_special_tokens=lowerCamelCase       )

   if token_ids_a is None:
     return [1] + ([0] * len(lowerCamelCase       )) + [1]
   return [1] + ([0] * len(lowerCamelCase       )) + [1, 1] + ([0] * len(lowerCamelCase       )) + [1]
 def 		lowercase_(   self			:    List[str]				,   lowerCamelCase			:    List[int]				,   lowerCamelCase			:    Optional[List[int]] = None       ):



   """simple docstring"""





   __A		:     int												=		[self.sep_token_id]
   __A		:     List[str]												=		[self.cls_token_id]

   if token_ids_a is None:
     return len(cls + token_ids_a + sep       ) * [0]
   return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]
 def 		lowercase_(   self			:    Dict				,   lowerCamelCase			:    int				,   lowerCamelCase			:    Dict=False				,   **lowerCamelCase			:    List[Any]       ):



   """simple docstring"""





   __A		:     Union[str, Any]												=		kwargs.pop("""add_prefix_space"""				,   self.add_prefix_space       )
   if (is_split_into_words or add_prefix_space) and (len(lowerCamelCase       ) > 0 and not text[0].isspace()):
     __A		:     Dict												=		""" """ + text
   return (text, kwargs)






 def 		lowercase_(   self			:    Optional[int]				,   lowerCamelCase			:    Union[Dict[str, EncodedInput], BatchEncoding]				,   lowerCamelCase			:    Optional[int] = None				,   lowerCamelCase			:    PaddingStrategy = PaddingStrategy.DO_NOT_PAD				,   lowerCamelCase			:    Optional[int] = None				,   lowerCamelCase			:    Optional[bool] = None				,   ):



   """simple docstring"""





   __A		:     str												=		super()._pad(
       encoded_inputs=lowerCamelCase				,   max_length=lowerCamelCase				,   padding_strategy=lowerCamelCase				,   pad_to_multiple_of=lowerCamelCase				,   return_attention_mask=lowerCamelCase				,   )

   # Load from model defaults
   if return_attention_mask is None:
     __A		:     List[Any]												=		"""attention_mask""" in self.model_input_names

   if return_attention_mask and "global_attention_mask" in encoded_inputs:
     __A		:     Tuple												=		encoded_inputs[self.model_input_names[0]]
     # `global_attention_mask` need to have the same length as other (sequential) inputs.
     __A		:     List[Any]												=		len(encoded_inputs["""global_attention_mask"""]       ) != len(lowerCamelCase       )

     if needs_to_be_padded:
       __A		:     str												=		len(lowerCamelCase       ) - len(encoded_inputs["""global_attention_mask"""]       )

       if self.padding_side == "right":
         # Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
         __A		:     List[Any]												=		(
             encoded_inputs["""global_attention_mask"""] + [-1] * difference
         )
       elif self.padding_side == "left":
         __A		:     Any												=		[-1] * difference + encoded_inputs[
             """global_attention_mask"""
         ]
       else:
         raise ValueError("""Invalid padding strategy:""" + str(self.padding_side       )       )

   return encoded_inputs



