

'''simple docstring'''





import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase__       = logging.get_logger(__name__)


lowerCAmelCase__       = {"vocab_file": "vocab.json", "merges_file": "merges.txt"}

lowerCAmelCase__       = {
    "vocab_file": {
        "allenai/longformer-base-4096": "https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json",
        "allenai/longformer-large-4096": (
            "https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"
        ),
        "allenai/longformer-large-4096-finetuned-triviaqa": (
            "https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"
        ),
        "allenai/longformer-base-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"
        ),
        "allenai/longformer-large-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"
        ),
    },
    "merges_file": {
        "allenai/longformer-base-4096": "https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt",
        "allenai/longformer-large-4096": (
            "https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"
        ),
        "allenai/longformer-large-4096-finetuned-triviaqa": (
            "https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"
        ),
        "allenai/longformer-base-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"
        ),
        "allenai/longformer-large-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"
        ),
    },
}

lowerCAmelCase__       = {
    "allenai/longformer-base-4096": 4096,
    "allenai/longformer-large-4096": 4096,
    "allenai/longformer-large-4096-finetuned-triviaqa": 4096,
    "allenai/longformer-base-4096-extra.pos.embd.only": 4096,
    "allenai/longformer-large-4096-extra.pos.embd.only": 4096,
}
@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def      _lowerCAmelCase(	):
  UpperCAmelCase					      =  (
      list(range(ord("!"			)   ,       ord("~"			) + 1			)			) + list(range(ord("¡"			)   ,       ord("¬"			) + 1			)			) + list(range(ord("®"			)   ,       ord("ÿ"			) + 1			)			)
  )
  UpperCAmelCase					      =  bs[:]
  UpperCAmelCase					      =  0
  for b in range(2**8			):
    if b not in bs:
      bs.append(__A			)
      cs.append(2**8 + n			)
      n += 1
  UpperCAmelCase					      =  [chr(__A			) for n in cs]
  return dict(zip(__A   ,       __A			)			)
def      _lowerCAmelCase(	__A			):
  UpperCAmelCase					      =  set()
  UpperCAmelCase					      =  word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)			)
    UpperCAmelCase					      =  char
  return pairs






class 						__magic_name__    (       _snake_case    ):
 UpperCAmelCase							    =						VOCAB_FILES_NAMES
 UpperCAmelCase							    =						PRETRAINED_VOCAB_FILES_MAP
 UpperCAmelCase							    =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCAmelCase							    =						["""input_ids""", """attention_mask"""]


 def __init__(						self		: List[str]			,						lowerCAmelCase__		: Any			,						lowerCAmelCase__		: str			,						lowerCAmelCase__		: Tuple="replace"			,						lowerCAmelCase__		: str="<s>"			,						lowerCAmelCase__		: int="</s>"			,						lowerCAmelCase__		: int="</s>"			,						lowerCAmelCase__		: Dict="<s>"			,						lowerCAmelCase__		: Dict="<unk>"			,						lowerCAmelCase__		: Optional[Any]="<pad>"			,						lowerCAmelCase__		: int="<mask>"			,						lowerCAmelCase__		: str=False			,						**lowerCAmelCase__		: List[Any]			,						)    ->      List[str]:
   UpperCAmelCase					      =  AddedToken(lowerCAmelCase__			,						lstrip=lowerCAmelCase__			,						rstrip=lowerCAmelCase__    ) if isinstance(lowerCAmelCase__			,						lowerCAmelCase__    ) else bos_token
   UpperCAmelCase					      =  AddedToken(lowerCAmelCase__			,						lstrip=lowerCAmelCase__			,						rstrip=lowerCAmelCase__    ) if isinstance(lowerCAmelCase__			,						lowerCAmelCase__    ) else eos_token
   UpperCAmelCase					      =  AddedToken(lowerCAmelCase__			,						lstrip=lowerCAmelCase__			,						rstrip=lowerCAmelCase__    ) if isinstance(lowerCAmelCase__			,						lowerCAmelCase__    ) else sep_token
   UpperCAmelCase					      =  AddedToken(lowerCAmelCase__			,						lstrip=lowerCAmelCase__			,						rstrip=lowerCAmelCase__    ) if isinstance(lowerCAmelCase__			,						lowerCAmelCase__    ) else cls_token
   UpperCAmelCase					      =  AddedToken(lowerCAmelCase__			,						lstrip=lowerCAmelCase__			,						rstrip=lowerCAmelCase__    ) if isinstance(lowerCAmelCase__			,						lowerCAmelCase__    ) else unk_token
   UpperCAmelCase					      =  AddedToken(lowerCAmelCase__			,						lstrip=lowerCAmelCase__			,						rstrip=lowerCAmelCase__    ) if isinstance(lowerCAmelCase__			,						lowerCAmelCase__    ) else pad_token

   # Mask token behave like a normal word, i.e. include the space before it
   UpperCAmelCase					      =  AddedToken(lowerCAmelCase__			,						lstrip=lowerCAmelCase__			,						rstrip=lowerCAmelCase__    ) if isinstance(lowerCAmelCase__			,						lowerCAmelCase__    ) else mask_token

   super().__init__(
       errors=lowerCAmelCase__			,						bos_token=lowerCAmelCase__			,						eos_token=lowerCAmelCase__			,						unk_token=lowerCAmelCase__			,						sep_token=lowerCAmelCase__			,						cls_token=lowerCAmelCase__			,						pad_token=lowerCAmelCase__			,						mask_token=lowerCAmelCase__			,						add_prefix_space=lowerCAmelCase__			,						**lowerCAmelCase__			,						)

   with open(lowerCAmelCase__			,						encoding="utf-8"    ) as vocab_handle:
     UpperCAmelCase					      =  json.load(lowerCAmelCase__    )
   UpperCAmelCase					      =  {v: k for k, v in self.encoder.items()}
   UpperCAmelCase					      =  errors  # how to handle errors in decoding
   UpperCAmelCase					      =  bytes_to_unicode()
   UpperCAmelCase					      =  {v: k for k, v in self.byte_encoder.items()}
   with open(lowerCAmelCase__			,						encoding="utf-8"    ) as merges_handle:
     UpperCAmelCase					      =  merges_handle.read().split("\n"    )[1:-1]
   UpperCAmelCase					      =  [tuple(merge.split()    ) for merge in bpe_merges]
   UpperCAmelCase					      =  dict(zip(lowerCAmelCase__			,						range(len(lowerCAmelCase__    )    )    )    )
   UpperCAmelCase					      =  {}
   UpperCAmelCase					      =  add_prefix_space

   # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
   UpperCAmelCase					      =  re.compile(r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"    )


 @property
 def _UpperCamelCase	(						self		: List[Any]    )    ->      Optional[int]:
   return len(self.encoder    )


 def _UpperCamelCase	(						self		: Optional[int]    )    ->      str:
   return dict(self.encoder			,						**self.added_tokens_encoder    )


 def _UpperCamelCase	(						self		: List[str]			,						lowerCAmelCase__		: Optional[int]    )    ->      int:
   if token in self.cache:
     return self.cache[token]
   UpperCAmelCase					      =  tuple(lowerCAmelCase__    )
   UpperCAmelCase					      =  get_pairs(lowerCAmelCase__    )

   if not pairs:
     return token

   while True:
     UpperCAmelCase					      =  min(lowerCAmelCase__			,						key=lambda lowerCAmelCase__    : self.bpe_ranks.get(lowerCAmelCase__			,						float("inf"    )    )    )
     if bigram not in self.bpe_ranks:
       break
     UpperCAmelCase      ,	UpperCAmelCase					      =  bigram
     UpperCAmelCase					      =  []
     UpperCAmelCase					      =  0
     while i < len(lowerCAmelCase__    ):
       try:
         UpperCAmelCase					      =  word.index(lowerCAmelCase__			,						lowerCAmelCase__    )
       except ValueError:
         new_word.extend(word[i:]    )
         break
       else:
         new_word.extend(word[i:j]    )
         UpperCAmelCase					      =  j

       if word[i] == first and i < len(lowerCAmelCase__    ) - 1 and word[i + 1] == second:
         new_word.append(first + second    )
         i += 2
       else:
         new_word.append(word[i]    )
         i += 1
     UpperCAmelCase					      =  tuple(lowerCAmelCase__    )
     UpperCAmelCase					      =  new_word
     if len(lowerCAmelCase__    ) == 1:
       break
     else:
       UpperCAmelCase					      =  get_pairs(lowerCAmelCase__    )
   UpperCAmelCase					      =  " ".join(lowerCAmelCase__    )
   UpperCAmelCase					      =  word
   return word


 def _UpperCamelCase	(						self		: List[Any]			,						lowerCAmelCase__		: List[str]    )    ->      str:
   UpperCAmelCase					      =  []
   for token in re.findall(self.pat			,						lowerCAmelCase__    ):
     UpperCAmelCase					      =  "".join(
         self.byte_encoder[b] for b in token.encode("utf-8"    )    )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
     bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCAmelCase__    ).split(" "    )    )
   return bpe_tokens


 def _UpperCamelCase	(						self		: str			,						lowerCAmelCase__		: List[str]    )    ->      str:
   return self.encoder.get(lowerCAmelCase__			,						self.encoder.get(self.unk_token    )    )


 def _UpperCamelCase	(						self		: Tuple			,						lowerCAmelCase__		: Dict    )    ->      Tuple:
   return self.decoder.get(lowerCAmelCase__    )


 def _UpperCamelCase	(						self		: List[str]			,						lowerCAmelCase__		: Dict    )    ->      Tuple:
   UpperCAmelCase					      =  "".join(lowerCAmelCase__    )
   UpperCAmelCase					      =  bytearray([self.byte_decoder[c] for c in text]    ).decode("utf-8"			,						errors=self.errors    )
   return text


 def _UpperCamelCase	(						self		: str			,						lowerCAmelCase__		: str			,						lowerCAmelCase__		: Optional[str] = None    )    ->      Tuple[str]:
   if not os.path.isdir(lowerCAmelCase__    ):
     logger.error(f"Vocabulary path ({save_directory}) should be a directory"    )
     return
   UpperCAmelCase					      =  os.path.join(
       lowerCAmelCase__			,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]    )
   UpperCAmelCase					      =  os.path.join(
       lowerCAmelCase__			,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]    )

   with open(lowerCAmelCase__			,						"w"			,						encoding="utf-8"    ) as f:
     f.write(json.dumps(self.encoder			,						indent=2			,						sort_keys=lowerCAmelCase__			,						ensure_ascii=lowerCAmelCase__    ) + "\n"    )

   UpperCAmelCase					      =  0
   with open(lowerCAmelCase__			,						"w"			,						encoding="utf-8"    ) as writer:
     writer.write("#version: 0.2\n"    )
     for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,						key=lambda lowerCAmelCase__    : kv[1]    ):
       if index != token_index:
         logger.warning(
             f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
             " Please check that the tokenizer is not corrupted!"    )
         UpperCAmelCase					      =  token_index
       writer.write(" ".join(lowerCAmelCase__    ) + "\n"    )
       index += 1

   return vocab_file, merge_file


 def _UpperCamelCase	(						self		: Any			,						lowerCAmelCase__		: List[int]			,						lowerCAmelCase__		: Optional[List[int]] = None    )    ->      List[int]:
   if token_ids_a is None:
     return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
   UpperCAmelCase					      =  [self.cls_token_id]
   UpperCAmelCase					      =  [self.sep_token_id]
   return cls + token_ids_a + sep + sep + token_ids_a + sep


 def _UpperCamelCase	(						self		: str			,						lowerCAmelCase__		: List[int]			,						lowerCAmelCase__		: Optional[List[int]] = None			,						lowerCAmelCase__		: bool = False    )    ->      List[int]:
   if already_has_special_tokens:
     return super().get_special_tokens_mask(
         token_ids_a=lowerCAmelCase__			,						token_ids_a=lowerCAmelCase__			,						already_has_special_tokens=lowerCAmelCase__    )

   if token_ids_a is None:
     return [1] + ([0] * len(lowerCAmelCase__    )) + [1]
   return [1] + ([0] * len(lowerCAmelCase__    )) + [1, 1] + ([0] * len(lowerCAmelCase__    )) + [1]


 def _UpperCamelCase	(						self		: Optional[int]			,						lowerCAmelCase__		: List[int]			,						lowerCAmelCase__		: Optional[List[int]] = None    )    ->      List[int]:
   UpperCAmelCase					      =  [self.sep_token_id]
   UpperCAmelCase					      =  [self.cls_token_id]

   if token_ids_a is None:
     return len(cls + token_ids_a + sep    ) * [0]
   return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]


 def _UpperCamelCase	(						self		: str			,						lowerCAmelCase__		: Optional[int]			,						lowerCAmelCase__		: Optional[int]=False			,						**lowerCAmelCase__		: List[Any]    )    ->      str:
   UpperCAmelCase					      =  kwargs.pop("add_prefix_space"			,						self.add_prefix_space    )
   if (is_split_into_words or add_prefix_space) and (len(lowerCAmelCase__    ) > 0 and not text[0].isspace()):
     UpperCAmelCase					      =  " " + text
   return (text, kwargs)






lowerCAmelCase__       = "\n# Transformers 설치 방법\n! pip install transformers datasets\n# 마지막 릴리스 대신 소스에서 설치하려면, 위 명령을 주석으로 바꾸고 아래 명령을 해제하세요.\n# ! pip install git+https://github.com/huggingface/transformers.git\n"

lowerCAmelCase__       = [{"type": "code", "content": INSTALL_CONTENT}]
lowerCAmelCase__       = {
    "{processor_class}": "FakeProcessorClass",
    "{model_class}": "FakeModelClass",
    "{object_class}": "FakeObjectClass",
}



