


import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


UpperCAmelCase_   :				List[str]						      = logging.get_logger(__name__)

UpperCAmelCase_   :				int						      = {'vocab_file': 'spiece.model'}

UpperCAmelCase_   :				Optional[int]						      = {
    'vocab_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model',
    }
}

UpperCAmelCase_   :				int						      = {
    'xlnet-base-cased': None,
    'xlnet-large-cased': None,
}

# Segments (not really needed)
UpperCAmelCase_   :				List[Any]						      = 0
UpperCAmelCase_   :				List[Any]						      = 1
UpperCAmelCase_   :				Any						      = 2
UpperCAmelCase_   :				int						      = 3
UpperCAmelCase_   :				Tuple						      = 4
class 				SCREAMING_SNAKE_CASE__						(       _A    ):
      snake_case__       :  Optional[int]       =		VOCAB_FILES_NAMES
      snake_case__       :  int       =		PRETRAINED_VOCAB_FILES_MAP
      snake_case__       :  Tuple       =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      snake_case__       :  int       =		'''left'''
      def __init__( self       : List[str]  ,							SCREAMING_SNAKE_CASE__       : Any  ,							SCREAMING_SNAKE_CASE__       : str=False  ,							SCREAMING_SNAKE_CASE__       : Dict=True  ,							SCREAMING_SNAKE_CASE__       : List[str]=False  ,							SCREAMING_SNAKE_CASE__       : List[str]="<s>"  ,							SCREAMING_SNAKE_CASE__       : int="</s>"  ,							SCREAMING_SNAKE_CASE__       : str="<unk>"  ,							SCREAMING_SNAKE_CASE__       : int="<sep>"  ,							SCREAMING_SNAKE_CASE__       : Optional[Any]="<pad>"  ,							SCREAMING_SNAKE_CASE__       : Tuple="<cls>"  ,							SCREAMING_SNAKE_CASE__       : Optional[Any]="<mask>"  ,							SCREAMING_SNAKE_CASE__       : int=["<eop>", "<eod>"]  ,							SCREAMING_SNAKE_CASE__       : Optional[Dict[str, Any]] = None  ,							**SCREAMING_SNAKE_CASE__       : Optional[int]  ,							)	->    None:
             # Mask token behave like a normal word, i.e. include the space before it
             a_      :	str							=						AddedToken(__lowerCamelCase  ,							lstrip=__lowerCamelCase  ,							rstrip=__lowerCamelCase  ) if isinstance(__lowerCamelCase  ,							__lowerCamelCase  ) else mask_token

             a_      :	Optional[int]							=						{} if sp_model_kwargs is None else sp_model_kwargs

             super().__init__(
                 do_lower_case=__lowerCamelCase  ,							remove_space=__lowerCamelCase  ,							keep_accents=__lowerCamelCase  ,							bos_token=__lowerCamelCase  ,							eos_token=__lowerCamelCase  ,							unk_token=__lowerCamelCase  ,							sep_token=__lowerCamelCase  ,							pad_token=__lowerCamelCase  ,							cls_token=__lowerCamelCase  ,							mask_token=__lowerCamelCase  ,							additional_special_tokens=__lowerCamelCase  ,							sp_model_kwargs=self.sp_model_kwargs  ,							**__lowerCamelCase  ,							)

             a_      :	int							=						3

             a_      :	Union[str, Any]							=						do_lower_case
             a_      :	List[Any]							=						remove_space
             a_      :	List[Any]							=						keep_accents
             a_      :	str							=						vocab_file

             a_      :	Optional[Any]							=						spm.SentencePieceProcessor(**self.sp_model_kwargs  )
             self.sp_model.Load(__lowerCamelCase  )
      @property
      def  SCREAMING_SNAKE_CASE							( self       : str  )	->    int:
             return len(self.sp_model  )
      def  SCREAMING_SNAKE_CASE							( self       : Union[str, Any]  )	->    Dict:
             a_      :	str							=						{self.convert_ids_to_tokens(__lowerCamelCase  ): i for i in range(self.vocab_size  )}
             vocab.update(self.added_tokens_encoder  )
             return vocab
      def __getstate__( self       : Tuple  )	->    Optional[Any]:
             a_      :	Optional[int]							=						self.__dict__.copy()
             a_      :	Dict							=						None
             return state
      def __setstate__( self       : Any  ,							SCREAMING_SNAKE_CASE__       : Any  )	->    Dict:
             a_      :	Union[str, Any]							=						d

             # for backward compatibility
             if not hasattr(self  ,							'sp_model_kwargs'  ):
                    a_      :	Optional[int]							=						{}

             a_      :	Optional[int]							=						spm.SentencePieceProcessor(**self.sp_model_kwargs  )
             self.sp_model.Load(self.vocab_file  )
      def  SCREAMING_SNAKE_CASE							( self       : Union[str, Any]  ,							SCREAMING_SNAKE_CASE__       : int  )	->    Any:
             if self.remove_space:
                    a_      :	Union[str, Any]							=						" ".join(inputs.strip().split()  )
             else:
                    a_      :	str							=						inputs
             a_      :	Dict							=						outputs.replace('``'  ,							'\"'  ).replace('\'\''  ,							'\"'  )

             if not self.keep_accents:
                    a_      :	Optional[int]							=						unicodedata.normalize('NFKD'  ,							__lowerCamelCase  )
                    a_      :	Tuple							=						"".join([c for c in outputs if not unicodedata.combining(__lowerCamelCase  )]  )
             if self.do_lower_case:
                    a_      :	Tuple							=						outputs.lower()

             return outputs
      def  SCREAMING_SNAKE_CASE							( self       : Optional[int]  ,							SCREAMING_SNAKE_CASE__       : str  )	->    List[str]:
             a_      :	Dict							=						self.preprocess_text(__lowerCamelCase  )
             a_      :	Tuple							=						self.sp_model.encode(__lowerCamelCase  ,							out_type=__lowerCamelCase  )
             a_      :	Any							=						[]
             for piece in pieces:
                    if len(__lowerCamelCase  ) > 1 and piece[-1] == str(','  ) and piece[-2].isdigit():
                           a_      :	Optional[Any]							=						self.sp_model.EncodeAsPieces(piece[:-1].replace(__lowerCamelCase  ,							''  )  )
                           if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                                  if len(cur_pieces[0]  ) == 1:
                                         a_      :	Optional[Any]							=						cur_pieces[1:]
                                  else:
                                         a_      :	List[str]							=						cur_pieces[0][1:]
                           cur_pieces.append(piece[-1]  )
                           new_pieces.extend(__lowerCamelCase  )
                    else:
                           new_pieces.append(__lowerCamelCase  )

             return new_pieces
      def  SCREAMING_SNAKE_CASE							( self       : Any  ,							SCREAMING_SNAKE_CASE__       : Tuple  )	->    Dict:
             return self.sp_model.PieceToId(__lowerCamelCase  )
      def  SCREAMING_SNAKE_CASE							( self       : int  ,							SCREAMING_SNAKE_CASE__       : Optional[int]  )	->    List[str]:
             return self.sp_model.IdToPiece(__lowerCamelCase  )
      def  SCREAMING_SNAKE_CASE							( self       : Optional[Any]  ,							SCREAMING_SNAKE_CASE__       : Tuple  )	->    Tuple:
             a_      :	Any							=						"".join(__lowerCamelCase  ).replace(__lowerCamelCase  ,							' '  ).strip()
             return out_string
      def  SCREAMING_SNAKE_CASE							( self       : Union[str, Any]  ,							SCREAMING_SNAKE_CASE__       : List[int]  ,							SCREAMING_SNAKE_CASE__       : bool = False  ,							SCREAMING_SNAKE_CASE__       : bool = None  ,							SCREAMING_SNAKE_CASE__       : bool = True  ,							**SCREAMING_SNAKE_CASE__       : Union[str, Any]  ,							)	->    str:
             a_      :	Any							=						kwargs.pop('use_source_tokenizer'  ,							__lowerCamelCase  )

             a_      :	Any							=						self.convert_ids_to_tokens(__lowerCamelCase  ,							skip_special_tokens=__lowerCamelCase  )

             # To avoid mixing byte-level and unicode for byte-level BPT
             # we need to build string separately for added tokens and byte-level tokens
             # cf. https://github.com/huggingface/transformers/issues/1133
             a_      :	Dict							=						[]
             a_      :	Dict							=						[]
             for token in filtered_tokens:
                    if skip_special_tokens and token in self.all_special_ids:
                           continue
                    if token in self.added_tokens_encoder:
                           if current_sub_text:
                                  sub_texts.append(self.convert_tokens_to_string(__lowerCamelCase  )  )
                                  a_      :	Optional[Any]							=						[]
                           sub_texts.append(__lowerCamelCase  )
                    else:
                           current_sub_text.append(__lowerCamelCase  )
             if current_sub_text:
                    sub_texts.append(self.convert_tokens_to_string(__lowerCamelCase  )  )

             # Mimic the behavior of the Rust tokenizer:
             # By default, there are no spaces between special tokens
             a_      :	Any							=						"".join(__lowerCamelCase  )

             a_      :	Optional[int]							=						(
                 clean_up_tokenization_spaces
                 if clean_up_tokenization_spaces is not None
                 else self.clean_up_tokenization_spaces
             )
             if clean_up_tokenization_spaces:
                    a_      :	Dict							=						self.clean_up_tokenization(__lowerCamelCase  )
                    return clean_text
             else:
                    return text
      def  SCREAMING_SNAKE_CASE							( self       : Any  ,							SCREAMING_SNAKE_CASE__       : List[int]  ,							SCREAMING_SNAKE_CASE__       : Optional[List[int]] = None  )	->    List[int]:
             a_      :	Dict							=						[self.sep_token_id]
             a_      :	Dict							=						[self.cls_token_id]
             if token_ids_a is None:
                    return token_ids_a + sep + cls
             return token_ids_a + sep + token_ids_a + sep + cls
      def  SCREAMING_SNAKE_CASE							( self       : str  ,							SCREAMING_SNAKE_CASE__       : List[int]  ,							SCREAMING_SNAKE_CASE__       : Optional[List[int]] = None  ,							SCREAMING_SNAKE_CASE__       : bool = False  )	->    List[int]:

             if already_has_special_tokens:
                    return super().get_special_tokens_mask(
                        token_ids_a=__lowerCamelCase  ,							token_ids_a=__lowerCamelCase  ,							already_has_special_tokens=__lowerCamelCase  )

             if token_ids_a is not None:
                    return ([0] * len(__lowerCamelCase  )) + [1] + ([0] * len(__lowerCamelCase  )) + [1, 1]
             return ([0] * len(__lowerCamelCase  )) + [1, 1]
      def  SCREAMING_SNAKE_CASE							( self       : List[Any]  ,							SCREAMING_SNAKE_CASE__       : List[int]  ,							SCREAMING_SNAKE_CASE__       : Optional[List[int]] = None  )	->    List[int]:
             a_      :	Optional[Any]							=						[self.sep_token_id]
             a_      :	Dict							=						[2]

             if token_ids_a is None:
                    return len(token_ids_a + sep  ) * [0] + cls_segment_id
             return len(token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1] + cls_segment_id

      def  SCREAMING_SNAKE_CASE							( self       : Union[str, Any]  ,							SCREAMING_SNAKE_CASE__       : str  ,							SCREAMING_SNAKE_CASE__       : Optional[str] = None  )	->    Tuple[str]:
             if not os.path.isdir(__lowerCamelCase  ):
                    logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""  )
                    return
             a_      :	str							=						os.path.join(
                 __lowerCamelCase  ,							(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']  )

             if os.path.abspath(self.vocab_file  ) != os.path.abspath(__lowerCamelCase  ) and os.path.isfile(self.vocab_file  ):
                    copyfile(self.vocab_file  ,							__lowerCamelCase  )
             elif not os.path.isfile(self.vocab_file  ):
                    with open(__lowerCamelCase  ,							'wb'  ) as fi:
                           a_      :	Tuple							=						self.sp_model.serialized_model_proto()
                           fi.write(__lowerCamelCase  )

             return (out_vocab_file,)






UpperCAmelCase_   :				Optional[int]						      = '\n# Transformers installation\n! pip install transformers datasets\n# To install from source instead of the last release, comment the command above and uncomment the following one.\n# ! pip install git+https://github.com/huggingface/transformers.git\n'

UpperCAmelCase_   :				List[str]						      = [{'type': 'code', 'content': INSTALL_CONTENT}]
UpperCAmelCase_   :				int						      = {
    '{processor_class}': 'FakeProcessorClass',
    '{model_class}': 'FakeModelClass',
    '{object_class}': 'FakeObjectClass',
}


