




'''simple docstring'''


import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


UpperCAmelCase_ 		=      logging.get_logger(__name__)

UpperCAmelCase_ 		=      {"vocab_file": "spiece.model"}

UpperCAmelCase_ 		=      {
    "vocab_file": {
        "xlnet-base-cased": "https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model",
        "xlnet-large-cased": "https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model",
    }
}

UpperCAmelCase_ 		=      {
    "xlnet-base-cased": None,
    "xlnet-large-cased": None,
}

# Segments (not really needed)
UpperCAmelCase_ 		=      0
UpperCAmelCase_ 		=      1
UpperCAmelCase_ 		=      2
UpperCAmelCase_ 		=      3
UpperCAmelCase_ 		=      4





class    __lowercase						(  __magic_name__ ):
   _a							=      VOCAB_FILES_NAMES
   _a							=      PRETRAINED_VOCAB_FILES_MAP
   _a							=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _a							=      """left"""







   def __init__(	self      ,							UpperCamelCase      ,							UpperCamelCase=False      ,							UpperCamelCase=True      ,							UpperCamelCase=False      ,							UpperCamelCase="<s>"      ,							UpperCamelCase="</s>"      ,							UpperCamelCase="<unk>"      ,							UpperCamelCase="<sep>"      ,							UpperCamelCase="<pad>"      ,							UpperCamelCase="<cls>"      ,							UpperCamelCase="<mask>"      ,							UpperCamelCase=["<eop>", "<eod>"]      ,							UpperCamelCase = None      ,							**UpperCamelCase      ,							)				->    None:
     # Mask token behave like a normal word, i.e. include the space before it
     __a			=					AddedToken(UpperCamelCase      ,							lstrip=UpperCamelCase      ,							rstrip=UpperCamelCase      ) if isinstance(UpperCamelCase      ,							UpperCamelCase      ) else mask_token

     __a			=					{} if sp_model_kwargs is None else sp_model_kwargs

     super().__init__(
         do_lower_case=UpperCamelCase      ,							remove_space=UpperCamelCase      ,							keep_accents=UpperCamelCase      ,							bos_token=UpperCamelCase      ,							eos_token=UpperCamelCase      ,							unk_token=UpperCamelCase      ,							sep_token=UpperCamelCase      ,							pad_token=UpperCamelCase      ,							cls_token=UpperCamelCase      ,							mask_token=UpperCamelCase      ,							additional_special_tokens=UpperCamelCase      ,							sp_model_kwargs=self.sp_model_kwargs      ,							**UpperCamelCase      ,							)

     __a			=					3

     __a			=					do_lower_case
     __a			=					remove_space
     __a			=					keep_accents
     __a			=					vocab_file

     __a			=					spm.SentencePieceProcessor(**self.sp_model_kwargs      )
     self.sp_model.Load(UpperCamelCase      )







   @property
   def 		UpperCamelCase__			(	self      )				->    Optional[Any]:
     return len(self.sp_model      )







   def 		UpperCamelCase__			(	self      )				->    str:
     __a			=					{self.convert_ids_to_tokens(UpperCamelCase      ): i for i in range(self.vocab_size      )}
     vocab.update(self.added_tokens_encoder      )
     return vocab







   def __getstate__(	self      )				->    Tuple:
     __a			=					self.__dict__.copy()
     __a			=					None
     return state







   def __setstate__(	self      ,							UpperCamelCase      )				->    Any:
     __a			=					d

     # for backward compatibility
     if not hasattr(self      ,							'sp_model_kwargs'      ):
       __a			=					{}

     __a			=					spm.SentencePieceProcessor(**self.sp_model_kwargs      )
     self.sp_model.Load(self.vocab_file      )







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      )				->    Union[str, Any]:
     if self.remove_space:
       __a			=					' '.join(inputs.strip().split()      )
     else:
       __a			=					inputs
     __a			=					outputs.replace('``'      ,							'"'      ).replace('\'\''      ,							'"'      )

     if not self.keep_accents:
       __a			=					unicodedata.normalize('NFKD'      ,							UpperCamelCase      )
       __a			=					''.join([c for c in outputs if not unicodedata.combining(UpperCamelCase      )]      )
     if self.do_lower_case:
       __a			=					outputs.lower()

     return outputs







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      )				->    List[str]:
     __a			=					self.preprocess_text(UpperCamelCase      )
     __a			=					self.sp_model.encode(UpperCamelCase      ,							out_type=UpperCamelCase      )
     __a			=					[]
     for piece in pieces:
       if len(UpperCamelCase      ) > 1 and piece[-1] == str(','      ) and piece[-2].isdigit():
         __a			=					self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCamelCase      ,							''      )      )
         if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
           if len(cur_pieces[0]      ) == 1:
             __a			=					cur_pieces[1:]
           else:
             __a			=					cur_pieces[0][1:]
         cur_pieces.append(piece[-1]      )
         new_pieces.extend(UpperCamelCase      )
       else:
         new_pieces.append(UpperCamelCase      )

     return new_pieces







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      )				->    Optional[Any]:
     return self.sp_model.PieceToId(UpperCamelCase      )







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      )				->    Tuple:
     return self.sp_model.IdToPiece(UpperCamelCase      )







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      )				->    Optional[int]:
     __a			=					''.join(UpperCamelCase      ).replace(UpperCamelCase      ,							' '      ).strip()
     return out_string







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      ,							UpperCamelCase = False      ,							UpperCamelCase = None      ,							UpperCamelCase = True      ,							**UpperCamelCase      ,							)				->    str:
     __a			=					kwargs.pop('use_source_tokenizer'      ,							UpperCamelCase      )

     __a			=					self.convert_ids_to_tokens(UpperCamelCase      ,							skip_special_tokens=UpperCamelCase      )

     # To avoid mixing byte-level and unicode for byte-level BPT
     # we need to build string separately for added tokens and byte-level tokens
     # cf. https://github.com/huggingface/transformers/issues/1133
     __a			=					[]
     __a			=					[]
     for token in filtered_tokens:
       if skip_special_tokens and token in self.all_special_ids:
         continue
       if token in self.added_tokens_encoder:
         if current_sub_text:
           sub_texts.append(self.convert_tokens_to_string(UpperCamelCase      )      )
           __a			=					[]
         sub_texts.append(UpperCamelCase      )
       else:
         current_sub_text.append(UpperCamelCase      )
     if current_sub_text:
       sub_texts.append(self.convert_tokens_to_string(UpperCamelCase      )      )

     # Mimic the behavior of the Rust tokenizer:
     # By default, there are no spaces between special tokens
     __a			=					''.join(UpperCamelCase      )

     __a			=					(
         clean_up_tokenization_spaces
         if clean_up_tokenization_spaces is not None
         else self.clean_up_tokenization_spaces
     )
     if clean_up_tokenization_spaces:
       __a			=					self.clean_up_tokenization(UpperCamelCase      )
       return clean_text
     else:
       return text







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      ,							UpperCamelCase = None      )				->    List[int]:
     __a			=					[self.sep_token_id]
     __a			=					[self.cls_token_id]
     if token_ids_a is None:
       return token_ids_a + sep + cls
     return token_ids_a + sep + token_ids_a + sep + cls







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      ,							UpperCamelCase = None      ,							UpperCamelCase = False      )				->    List[int]:

     if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=UpperCamelCase      ,							token_ids_a=UpperCamelCase      ,							already_has_special_tokens=UpperCamelCase      )

     if token_ids_a is not None:
       return ([0] * len(UpperCamelCase      )) + [1] + ([0] * len(UpperCamelCase      )) + [1, 1]
     return ([0] * len(UpperCamelCase      )) + [1, 1]







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      ,							UpperCamelCase = None      )				->    List[int]:
     __a			=					[self.sep_token_id]
     __a			=					[2]

     if token_ids_a is None:
       return len(token_ids_a + sep      ) * [0] + cls_segment_id
     return len(token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1] + cls_segment_id







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      ,							UpperCamelCase = None      )				->    Tuple[str]:
     if not os.path.isdir(UpperCamelCase      ):
       logger.error(f"Vocabulary path ({save_directory}) should be a directory"      )
       return
     __a			=					os.path.join(
         UpperCamelCase      ,							(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']      )

     if os.path.abspath(self.vocab_file      ) != os.path.abspath(UpperCamelCase      ) and os.path.isfile(self.vocab_file      ):
       copyfile(self.vocab_file      ,							UpperCamelCase      )
     elif not os.path.isfile(self.vocab_file      ):
       with open(UpperCamelCase      ,							'wb'      ) as fi:
         __a			=					self.sp_model.serialized_model_proto()
         fi.write(UpperCamelCase      )

     return (out_vocab_file,)









'''simple docstring'''


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mobilebert import MobileBertTokenizer


UpperCAmelCase_ 		=      logging.get_logger(__name__)

UpperCAmelCase_ 		=      {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

UpperCAmelCase_ 		=      {
    "vocab_file": {"mobilebert-uncased": "https://huggingface.co/google/mobilebert-uncased/resolve/main/vocab.txt"},
    "tokenizer_file": {
        "mobilebert-uncased": "https://huggingface.co/google/mobilebert-uncased/resolve/main/tokenizer.json"
    },
}

UpperCAmelCase_ 		=      {"mobilebert-uncased": 5_12}


UpperCAmelCase_ 		=      {}





class    __lowercase						(  __magic_name__ ):
   _a							=      VOCAB_FILES_NAMES
   _a							=      PRETRAINED_VOCAB_FILES_MAP
   _a							=      PRETRAINED_INIT_CONFIGURATION
   _a							=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _a							=      MobileBertTokenizer







   def __init__(	self      ,							UpperCamelCase=None      ,							UpperCamelCase=None      ,							UpperCamelCase=True      ,							UpperCamelCase="[UNK]"      ,							UpperCamelCase="[SEP]"      ,							UpperCamelCase="[PAD]"      ,							UpperCamelCase="[CLS]"      ,							UpperCamelCase="[MASK]"      ,							UpperCamelCase=True      ,							UpperCamelCase=None      ,							**UpperCamelCase      ,							)				->    List[Any]:
     super().__init__(
         UpperCamelCase      ,							tokenizer_file=UpperCamelCase      ,							do_lower_case=UpperCamelCase      ,							unk_token=UpperCamelCase      ,							sep_token=UpperCamelCase      ,							pad_token=UpperCamelCase      ,							cls_token=UpperCamelCase      ,							mask_token=UpperCamelCase      ,							tokenize_chinese_chars=UpperCamelCase      ,							strip_accents=UpperCamelCase      ,							**UpperCamelCase      ,							)

     __a			=					json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
     if (
         normalizer_state.get('lowercase'      ,							UpperCamelCase      ) != do_lower_case
         or normalizer_state.get('strip_accents'      ,							UpperCamelCase      ) != strip_accents
         or normalizer_state.get('handle_chinese_chars'      ,							UpperCamelCase      ) != tokenize_chinese_chars
     ):
       __a			=					getattr(UpperCamelCase      ,							normalizer_state.pop('type'      )      )
       __a			=					do_lower_case
       __a			=					strip_accents
       __a			=					tokenize_chinese_chars
       __a			=					normalizer_class(**UpperCamelCase      )

     __a			=					do_lower_case







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      ,							UpperCamelCase=None      )				->    Tuple:
     __a			=					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

     if token_ids_a:
       output += token_ids_a + [self.sep_token_id]

     return output







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      ,							UpperCamelCase = None      )				->    List[int]:
     __a			=					[self.sep_token_id]
     __a			=					[self.cls_token_id]
     if token_ids_a is None:
       return len(cls + token_ids_a + sep      ) * [0]
     return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      ,							UpperCamelCase = None      )				->    Tuple[str]:
     __a			=					self._tokenizer.model.save(UpperCamelCase      ,							name=UpperCamelCase      )
     return tuple(UpperCamelCase      )



