

import os
import re
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_SCREAMING_SNAKE_CASE  :						Any         =    logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE  :						Tuple         =    {'''vocab_file''': '''spiece.model'''}

_SCREAMING_SNAKE_CASE  :						Dict         =    {
    '''vocab_file''': {
        '''google/bigbird-roberta-base''': '''https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model''',
        '''google/bigbird-roberta-large''': (
            '''https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model'''
        ),
        '''google/bigbird-base-trivia-itc''': (
            '''https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model'''
        ),
    }
}

_SCREAMING_SNAKE_CASE  :						Optional[int]         =    {
    '''google/bigbird-roberta-base''': 4096,
    '''google/bigbird-roberta-large''': 4096,
    '''google/bigbird-base-trivia-itc''': 4096,
}






class  UpperCAmelCase__	( A__ ):






  """simple docstring"""


  a     =      VOCAB_FILES_NAMES
  a     =      PRETRAINED_VOCAB_FILES_MAP
  a     =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  a     =      ["input_ids", "attention_mask"]
  a     =      []






  def __init__(							self		:	List[str]			,     __lowerCamelCase		:	Union[str, Any]			,     __lowerCamelCase		:	int="<unk>"			,     __lowerCamelCase		:	str="<s>"			,     __lowerCamelCase		:	List[str]="</s>"			,     __lowerCamelCase		:	List[Any]="<pad>"			,     __lowerCamelCase		:	Any="[SEP]"			,     __lowerCamelCase		:	str="[MASK]"			,     __lowerCamelCase		:	Optional[int]="[CLS]"			,     __lowerCamelCase		:	Optional[Dict[str, Any]] = None			,     **__lowerCamelCase		:	Union[str, Any]			,     )	->							None:
       SCREAMING_SNAKE_CASE__					=					AddedToken(__lowerCamelCase			,     lstrip=__lowerCamelCase			,     rstrip=__lowerCamelCase      ) if isinstance(__lowerCamelCase			,     __lowerCamelCase      ) else bos_token
       SCREAMING_SNAKE_CASE__					=					AddedToken(__lowerCamelCase			,     lstrip=__lowerCamelCase			,     rstrip=__lowerCamelCase      ) if isinstance(__lowerCamelCase			,     __lowerCamelCase      ) else eos_token
       SCREAMING_SNAKE_CASE__					=					AddedToken(__lowerCamelCase			,     lstrip=__lowerCamelCase			,     rstrip=__lowerCamelCase      ) if isinstance(__lowerCamelCase			,     __lowerCamelCase      ) else unk_token
       SCREAMING_SNAKE_CASE__					=					AddedToken(__lowerCamelCase			,     lstrip=__lowerCamelCase			,     rstrip=__lowerCamelCase      ) if isinstance(__lowerCamelCase			,     __lowerCamelCase      ) else pad_token
       SCREAMING_SNAKE_CASE__					=					AddedToken(__lowerCamelCase			,     lstrip=__lowerCamelCase			,     rstrip=__lowerCamelCase      ) if isinstance(__lowerCamelCase			,     __lowerCamelCase      ) else cls_token
       SCREAMING_SNAKE_CASE__					=					AddedToken(__lowerCamelCase			,     lstrip=__lowerCamelCase			,     rstrip=__lowerCamelCase      ) if isinstance(__lowerCamelCase			,     __lowerCamelCase      ) else sep_token

       # Mask token behave like a normal word, i.e. include the space before it
       SCREAMING_SNAKE_CASE__					=					AddedToken(__lowerCamelCase			,     lstrip=__lowerCamelCase			,     rstrip=__lowerCamelCase      ) if isinstance(__lowerCamelCase			,     __lowerCamelCase      ) else mask_token

       SCREAMING_SNAKE_CASE__					=					{} if sp_model_kwargs is None else sp_model_kwargs

       super().__init__(
           bos_token=__lowerCamelCase			,     eos_token=__lowerCamelCase			,     unk_token=__lowerCamelCase			,     pad_token=__lowerCamelCase			,     sep_token=__lowerCamelCase			,     mask_token=__lowerCamelCase			,     cls_token=__lowerCamelCase			,     sp_model_kwargs=self.sp_model_kwargs			,     **__lowerCamelCase			,     )

       SCREAMING_SNAKE_CASE__					=					vocab_file

       SCREAMING_SNAKE_CASE__					=					spm.SentencePieceProcessor(**self.sp_model_kwargs      )
       self.sp_model.Load(__lowerCamelCase      )






  @property
  def    lowercase_	(							self		:	Tuple      )	->							Tuple:
       return self.sp_model.get_piece_size()






  def    lowercase_	(							self		:	Dict      )	->							List[Any]:
       SCREAMING_SNAKE_CASE__					=					{self.convert_ids_to_tokens(__lowerCamelCase      ): i for i in range(self.vocab_size      )}
       vocab.update(self.added_tokens_encoder      )
       return vocab






  def __getstate__(							self		:	Optional[Any]      )	->							Any:
       SCREAMING_SNAKE_CASE__					=					self.__dict__.copy()
       SCREAMING_SNAKE_CASE__					=					None
       return state






  def __setstate__(							self		:	int			,     __lowerCamelCase		:	int      )	->							Tuple:
       SCREAMING_SNAKE_CASE__					=					d

       # for backward compatibility
       if not hasattr(self			,     '''sp_model_kwargs'''      ):
            SCREAMING_SNAKE_CASE__					=					{}

       SCREAMING_SNAKE_CASE__					=					spm.SentencePieceProcessor(**self.sp_model_kwargs      )
       self.sp_model.Load(self.vocab_file      )






  def    lowercase_	(							self		:	Any			,     __lowerCamelCase		:	str      )	->							List[str]:
       return self.sp_model.encode(__lowerCamelCase			,     out_type=__lowerCamelCase      )






  def    lowercase_	(							self		:	Dict			,     __lowerCamelCase		:	List[Any]      )	->							Optional[int]:
       return self.sp_model.piece_to_id(__lowerCamelCase      )






  def    lowercase_	(							self		:	str			,     __lowerCamelCase		:	Union[str, Any]      )	->							Any:
       SCREAMING_SNAKE_CASE__					=					self.sp_model.IdToPiece(__lowerCamelCase      )
       return token






  def    lowercase_	(							self		:	Tuple			,     __lowerCamelCase		:	Optional[Any]      )	->							Optional[Any]:
       SCREAMING_SNAKE_CASE__					=					[]
       SCREAMING_SNAKE_CASE__					=					''''''
       SCREAMING_SNAKE_CASE__					=					False
       for token in tokens:
            # make sure that special tokens are not decoded using sentencepiece model
            if token in self.all_special_tokens:
                 if not prev_is_special:
                      out_string += " "
                 out_string += self.sp_model.decode(__lowerCamelCase      ) + token
                 SCREAMING_SNAKE_CASE__					=					True
                 SCREAMING_SNAKE_CASE__					=					[]
            else:
                 current_sub_tokens.append(__lowerCamelCase      )
                 SCREAMING_SNAKE_CASE__					=					False
       out_string += self.sp_model.decode(__lowerCamelCase      )
       return out_string.strip()






  def    lowercase_	(							self		:	Optional[int]			,     __lowerCamelCase		:	List[int]			,     __lowerCamelCase		:	bool = False			,     __lowerCamelCase		:	bool = None			,     __lowerCamelCase		:	bool = True			,     **__lowerCamelCase		:	Union[str, Any]			,     )	->							str:
       SCREAMING_SNAKE_CASE__					=					kwargs.pop('''use_source_tokenizer'''			,     __lowerCamelCase      )

       SCREAMING_SNAKE_CASE__					=					self.convert_ids_to_tokens(__lowerCamelCase			,     skip_special_tokens=__lowerCamelCase      )

       # To avoid mixing byte-level and unicode for byte-level BPT
       # we need to build string separately for added tokens and byte-level tokens
       # cf. https://github.com/huggingface/transformers/issues/1133
       SCREAMING_SNAKE_CASE__					=					[]
       SCREAMING_SNAKE_CASE__					=					[]
       for token in filtered_tokens:
            if skip_special_tokens and token in self.all_special_ids:
                 continue
            if token in self.added_tokens_encoder:
                 if current_sub_text:
                      sub_texts.append(self.convert_tokens_to_string(__lowerCamelCase      )      )
                      SCREAMING_SNAKE_CASE__					=					[]
                 sub_texts.append(__lowerCamelCase      )
            else:
                 current_sub_text.append(__lowerCamelCase      )
       if current_sub_text:
            sub_texts.append(self.convert_tokens_to_string(__lowerCamelCase      )      )

       # Mimic the behavior of the Rust tokenizer:
       # No space before [MASK] and [SEP]
       if spaces_between_special_tokens:
            SCREAMING_SNAKE_CASE__					=					re.sub(r''' (\[(MASK|SEP)\])'''			,     r'''\1'''			,     ''' '''.join(__lowerCamelCase      )      )
       else:
            SCREAMING_SNAKE_CASE__					=					''''''.join(__lowerCamelCase      )

       SCREAMING_SNAKE_CASE__					=					(
           clean_up_tokenization_spaces
           if clean_up_tokenization_spaces is not None
           else self.clean_up_tokenization_spaces
       )
       if clean_up_tokenization_spaces:
            SCREAMING_SNAKE_CASE__					=					self.clean_up_tokenization(__lowerCamelCase      )
            return clean_text
       else:
            return text






  def    lowercase_	(							self		:	List[Any]			,     __lowerCamelCase		:	str			,     __lowerCamelCase		:	Optional[str] = None      )	->							Tuple[str]:
       if not os.path.isdir(__lowerCamelCase      ):
            logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''      )
            return
       SCREAMING_SNAKE_CASE__					=					os.path.join(
           __lowerCamelCase			,     (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']      )

       if os.path.abspath(self.vocab_file      ) != os.path.abspath(__lowerCamelCase      ) and os.path.isfile(self.vocab_file      ):
            copyfile(self.vocab_file			,     __lowerCamelCase      )
       elif not os.path.isfile(self.vocab_file      ):
            with open(__lowerCamelCase			,     '''wb'''      ) as fi:
                 SCREAMING_SNAKE_CASE__					=					self.sp_model.serialized_model_proto()
                 fi.write(__lowerCamelCase      )

       return (out_vocab_file,)






  def    lowercase_	(							self		:	List[str]			,     __lowerCamelCase		:	List[int]			,     __lowerCamelCase		:	Optional[List[int]] = None      )	->							List[int]:
       if token_ids_a is None:
            return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
       SCREAMING_SNAKE_CASE__					=					[self.cls_token_id]
       SCREAMING_SNAKE_CASE__					=					[self.sep_token_id]
       return cls + token_ids_a + sep + token_ids_a + sep






  def    lowercase_	(							self		:	Dict			,     __lowerCamelCase		:	List[int]			,     __lowerCamelCase		:	Optional[List[int]] = None			,     __lowerCamelCase		:	bool = False      )	->							List[int]:
       if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_a=__lowerCamelCase			,     token_ids_a=__lowerCamelCase			,     already_has_special_tokens=__lowerCamelCase      )

       if token_ids_a is None:
            return [1] + ([0] * len(__lowerCamelCase      )) + [1]
       return [1] + ([0] * len(__lowerCamelCase      )) + [1] + ([0] * len(__lowerCamelCase      )) + [1]







  def    lowercase_	(							self		:	Optional[Any]			,     __lowerCamelCase		:	List[int]			,     __lowerCamelCase		:	Optional[List[int]] = None      )	->							List[int]:
       SCREAMING_SNAKE_CASE__					=					[self.sep_token_id]
       SCREAMING_SNAKE_CASE__					=					[self.cls_token_id]
       if token_ids_a is None:
            return len(cls + token_ids_a + sep      ) * [0]
       return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]





from importlib import import_module

from .logging import get_logger


_SCREAMING_SNAKE_CASE  :						Optional[int]         =    get_logger(__name__)






class  UpperCAmelCase__	:






  """simple docstring"""


  def __init__(							self		:	Optional[int]			,     __lowerCamelCase		:	Optional[Any]			,     __lowerCamelCase		:	Any=None      )	->							int:
       SCREAMING_SNAKE_CASE__					=					attrs or []
       if module is not None:
            for key in module.__dict__:
                 if key in attrs or not key.startswith('''__'''      ):
                      setattr(self			,     __lowerCamelCase			,     getattr(__lowerCamelCase			,     __lowerCamelCase      )      )
       SCREAMING_SNAKE_CASE__					=					module._original_module if isinstance(__lowerCamelCase			,     _PatchedModuleObj      ) else module






class  UpperCAmelCase__	:






  """simple docstring"""


  a     =      []






  def __init__(							self		:	int			,     __lowerCamelCase		:	Union[str, Any]			,     __lowerCamelCase		:	str			,     __lowerCamelCase		:	Optional[int]			,     __lowerCamelCase		:	List[Any]=None      )	->							Any:
       SCREAMING_SNAKE_CASE__					=					obj
       SCREAMING_SNAKE_CASE__					=					target
       SCREAMING_SNAKE_CASE__					=					new
       SCREAMING_SNAKE_CASE__					=					target.split('''.'''      )[0]
       SCREAMING_SNAKE_CASE__					=					{}
       SCREAMING_SNAKE_CASE__					=					attrs or []






  def __enter__(							self		:	int      )	->							Tuple:
       *SCREAMING_SNAKE_CASE__,SCREAMING_SNAKE_CASE__					=					self.target.split('''.'''      )

       # Patch modules:
       # it's used to patch attributes of submodules like "os.path.join";
       # in this case we need to patch "os" and "os.path"

       for i in range(len(__lowerCamelCase      )      ):
            try:
                 SCREAMING_SNAKE_CASE__					=					import_module('''.'''.join(submodules[: i + 1]      )      )
            except ModuleNotFoundError:
                 continue
            # We iterate over all the globals in self.obj in case we find "os" or "os.path"
            for attr in self.obj.__dir__():
                 SCREAMING_SNAKE_CASE__					=					getattr(self.obj			,     __lowerCamelCase      )
                 # We don't check for the name of the global, but rather if its value *is* "os" or "os.path".
                 # This allows to patch renamed modules like "from os import path as ospath".
                 if obj_attr is submodule or (
                     (isinstance(__lowerCamelCase			,     _PatchedModuleObj      ) and obj_attr._original_module is submodule)
                 ):
                      SCREAMING_SNAKE_CASE__					=					obj_attr
                      # patch at top level
                      setattr(self.obj			,     __lowerCamelCase			,     _PatchedModuleObj(__lowerCamelCase			,     attrs=self.attrs      )      )
                      SCREAMING_SNAKE_CASE__					=					getattr(self.obj			,     __lowerCamelCase      )
                      # construct lower levels patches
                      for key in submodules[i + 1 :]:
                           setattr(__lowerCamelCase			,     __lowerCamelCase			,     _PatchedModuleObj(getattr(__lowerCamelCase			,     __lowerCamelCase			,     __lowerCamelCase      )			,     attrs=self.attrs      )      )
                           SCREAMING_SNAKE_CASE__					=					getattr(__lowerCamelCase			,     __lowerCamelCase      )
                      # finally set the target attribute
                      setattr(__lowerCamelCase			,     __lowerCamelCase			,     self.new      )

        # Patch attribute itself:
        # it's used for builtins like "open",
        # and also to patch "os.path.join" we may also need to patch "join"
        # itself if it was imported as "from os.path import join".

       if submodules:  # if it's an attribute of a submodule like "os.path.join"
            try:
                 SCREAMING_SNAKE_CASE__					=					getattr(import_module('''.'''.join(__lowerCamelCase      )      )			,     __lowerCamelCase      )
            except (AttributeError, ModuleNotFoundError):
                 return
            # We iterate over all the globals in self.obj in case we find "os.path.join"
            for attr in self.obj.__dir__():
                 # We don't check for the name of the global, but rather if its value *is* "os.path.join".
                 # This allows to patch renamed attributes like "from os.path import join as pjoin".
                 if getattr(self.obj			,     __lowerCamelCase      ) is attr_value:
                      SCREAMING_SNAKE_CASE__					=					getattr(self.obj			,     __lowerCamelCase      )
                      setattr(self.obj			,     __lowerCamelCase			,     self.new      )
       elif target_attr in globals()["__builtins__"]:  # if it'a s builtin like "open"
            SCREAMING_SNAKE_CASE__					=					globals()['''__builtins__'''][target_attr]
            setattr(self.obj			,     __lowerCamelCase			,     self.new      )
       else:
            raise RuntimeError(f'''Tried to patch attribute {target_attr} instead of a submodule.'''      )






  def __exit__(							self		:	Optional[Any]			,     *__lowerCamelCase		:	Tuple      )	->							List[str]:
       for attr in list(self.original      ):
            setattr(self.obj			,     __lowerCamelCase			,     self.original.pop(__lowerCamelCase      )      )






  def    lowercase_	(							self		:	List[Any]      )	->							Optional[Any]:
       self.__enter__()
       self._active_patches.append(self      )







  def    lowercase_	(							self		:	int      )	->							int:
       try:
            self._active_patches.remove(self      )
       except ValueError:
            # If the patch hasn't been started this will fail
            return None

       return self.__exit__()


