'''simple docstring'''




import collections
import importlib.util
import os
import re
from pathlib import Path


_A								=   'src/transformers'


# Matches is_xxx_available()
_A								=   re.compile(R'is\_([a-z_]*)_available()')
# Catches a one-line _import_struct = {xxx}
_A								=   re.compile(R'^_import_structure\s+=\s+\{([^\}]+)\}')
# Catches a line with a key-values pattern: "bla": ["foo", "bar"]
_A								=   re.compile(R'\s+"\S*":\s+\[([^\]]*)\]')
# Catches a line if not is_foo_available
_A								=   re.compile(R'^\s*if\s+not\s+is\_[a-z_]*\_available\(\)')
# Catches a line _import_struct["bla"].append("foo")
_A								=   re.compile(R'^\s*_import_structure\["\S*"\]\.append\("(\S*)"\)')
# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]
_A								=   re.compile(R'^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]')
# Catches a line with an object between quotes and a comma:     "MyModel",
_A								=   re.compile('^\s+"([^"]+)",')
# Catches a line with objects between brackets only:    ["foo", "bar"],
_A								=   re.compile('^\s+\[([^\]]+)\]')
# Catches a line with from foo import bar, bla, boo
_A								=   re.compile(R'\s+from\s+\S*\s+import\s+([^\(\s].*)\n')
# Catches a line with try:
_A								=   re.compile(R'^\s*try:')
# Catches a line with else:
_A								=   re.compile(R'^\s*else:')

def _UpperCamelCase   (       SCREAMING_SNAKE_CASE_							):
     if _re_test_backend.search(SCREAMING_SNAKE_CASE_							) is None:
          return None
     lowercase_					:       Optional[int]     				=					[b[0] for b in _re_backend.findall(SCREAMING_SNAKE_CASE_							)]
     backends.sort()
     return "_and_".join(SCREAMING_SNAKE_CASE_							)

def _UpperCamelCase   (       SCREAMING_SNAKE_CASE_							):
     with open(SCREAMING_SNAKE_CASE_  ,       'r'  ,       encoding='utf-8'  ,       newline='\n'							) as f:
          lowercase_					:       List[str]     				=					f.readlines()

     lowercase_					:       List[str]     				=					0
     while line_index < len(SCREAMING_SNAKE_CASE_							) and not lines[line_index].startswith('_import_structure = {'							):
          line_index += 1

     # If this is a traditional init, just return.
     if line_index >= len(SCREAMING_SNAKE_CASE_							):
          return None

     # First grab the objects without a specific backend in _import_structure
     lowercase_					:       int     				=					[]
     while not lines[line_index].startswith('if TYPE_CHECKING'							) and find_backend(lines[line_index]							) is None:
          lowercase_					:       str     				=					lines[line_index]
          # If we have everything on a single line, let's deal with it.
          if _re_one_line_import_struct.search(SCREAMING_SNAKE_CASE_							):
               lowercase_					:       Union[str, Any]     				=					_re_one_line_import_struct.search(SCREAMING_SNAKE_CASE_							).groups()[0]
               lowercase_					:       List[str]     				=					re.findall('\[([^\]]+)\]'  ,       SCREAMING_SNAKE_CASE_							)
               for imp in imports:
                    objects.extend([obj[1:-1] for obj in imp.split(', '							)]							)
               line_index += 1
               continue
          lowercase_					:       int     				=					_re_import_struct_key_value.search(SCREAMING_SNAKE_CASE_							)
          if single_line_import_search is not None:
               lowercase_					:       Optional[int]     				=					[obj[1:-1] for obj in single_line_import_search.groups()[0].split(', '							) if len(SCREAMING_SNAKE_CASE_							) > 0]
               objects.extend(SCREAMING_SNAKE_CASE_							)
          elif line.startswith(' ' * 8 + '"'							):
               objects.append(line[9:-3]							)
          line_index += 1

     lowercase_					:       List[Any]     				=					{'none': objects}
     # Let's continue with backend-specific objects in _import_structure
     while not lines[line_index].startswith('if TYPE_CHECKING'							):
          # If the line is an if not is_backend_available, we grab all objects associated.
          lowercase_					:       str     				=					find_backend(lines[line_index]							)
          # Check if the backend declaration is inside a try block:
          if _re_try.search(lines[line_index - 1]							) is None:
               lowercase_					:       Union[str, Any]     				=					None

          if backend is not None:
               line_index += 1

               # Scroll until we hit the else block of try-except-else
               while _re_else.search(lines[line_index]							) is None:
                    line_index += 1

               line_index += 1

               lowercase_					:       str     				=					[]
               # Until we unindent, add backend objects to the list
               while len(lines[line_index]							) <= 1 or lines[line_index].startswith(' ' * 4							):
                    lowercase_					:       Tuple     				=					lines[line_index]
                    if _re_import_struct_add_one.search(SCREAMING_SNAKE_CASE_							) is not None:
                         objects.append(_re_import_struct_add_one.search(SCREAMING_SNAKE_CASE_							).groups()[0]							)
                    elif _re_import_struct_add_many.search(SCREAMING_SNAKE_CASE_							) is not None:
                         lowercase_					:       str     				=					_re_import_struct_add_many.search(SCREAMING_SNAKE_CASE_							).groups()[0].split(', '							)
                         lowercase_					:       Optional[int]     				=					[obj[1:-1] for obj in imports if len(SCREAMING_SNAKE_CASE_							) > 0]
                         objects.extend(SCREAMING_SNAKE_CASE_							)
                    elif _re_between_brackets.search(SCREAMING_SNAKE_CASE_							) is not None:
                         lowercase_					:       Tuple     				=					_re_between_brackets.search(SCREAMING_SNAKE_CASE_							).groups()[0].split(', '							)
                         lowercase_					:       Any     				=					[obj[1:-1] for obj in imports if len(SCREAMING_SNAKE_CASE_							) > 0]
                         objects.extend(SCREAMING_SNAKE_CASE_							)
                    elif _re_quote_object.search(SCREAMING_SNAKE_CASE_							) is not None:
                         objects.append(_re_quote_object.search(SCREAMING_SNAKE_CASE_							).groups()[0]							)
                    elif line.startswith(' ' * 8 + '"'							):
                         objects.append(line[9:-3]							)
                    elif line.startswith(' ' * 12 + '"'							):
                         objects.append(line[13:-3]							)
                    line_index += 1

               lowercase_					:       Union[str, Any]     				=					objects
          else:
               line_index += 1

    # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend
     lowercase_					:       Optional[int]     				=					[]
     while (
         line_index < len(SCREAMING_SNAKE_CASE_							)
         and find_backend(lines[line_index]							) is None
         and not lines[line_index].startswith('else'							)
     ):
          lowercase_					:       Dict     				=					lines[line_index]
          lowercase_					:       Dict     				=					_re_import.search(SCREAMING_SNAKE_CASE_							)
          if single_line_import_search is not None:
               objects.extend(single_line_import_search.groups()[0].split(', '							)							)
          elif line.startswith(' ' * 8							):
               objects.append(line[8:-2]							)
          line_index += 1

     lowercase_					:       Any     				=					{'none': objects}
     # Let's continue with backend-specific objects
     while line_index < len(SCREAMING_SNAKE_CASE_							):
          # If the line is an if is_backend_available, we grab all objects associated.
          lowercase_					:       str     				=					find_backend(lines[line_index]							)
          # Check if the backend declaration is inside a try block:
          if _re_try.search(lines[line_index - 1]							) is None:
               lowercase_					:       int     				=					None

          if backend is not None:
               line_index += 1

               # Scroll until we hit the else block of try-except-else
               while _re_else.search(lines[line_index]							) is None:
                    line_index += 1

               line_index += 1

               lowercase_					:       int     				=					[]
               # Until we unindent, add backend objects to the list
               while len(lines[line_index]							) <= 1 or lines[line_index].startswith(' ' * 8							):
                    lowercase_					:       Tuple     				=					lines[line_index]
                    lowercase_					:       Union[str, Any]     				=					_re_import.search(SCREAMING_SNAKE_CASE_							)
                    if single_line_import_search is not None:
                         objects.extend(single_line_import_search.groups()[0].split(', '							)							)
                    elif line.startswith(' ' * 12							):
                         objects.append(line[12:-2]							)
                    line_index += 1

               lowercase_					:       List[str]     				=					objects
          else:
               line_index += 1

     return import_dict_objects, type_hint_objects

def _UpperCamelCase   (       SCREAMING_SNAKE_CASE_  ,       SCREAMING_SNAKE_CASE_							):

     def find_duplicates(SCREAMING_SNAKE_CASE_							):
          return [k for k, v in collections.Counter(SCREAMING_SNAKE_CASE_							).items() if v > 1]

     if list(import_dict_objects.keys()							) != list(type_hint_objects.keys()							):
          return ["Both sides of the init do not have the same backends!"]

     lowercase_					:       Any     				=					[]
     for key in import_dict_objects.keys():
          lowercase_					:       int     				=					find_duplicates(import_dict_objects[key]							)
          if duplicate_imports:
               errors.append(f'''Duplicate _import_structure definitions for: {duplicate_imports}'''							)
          lowercase_					:       Union[str, Any]     				=					find_duplicates(type_hint_objects[key]							)
          if duplicate_type_hints:
               errors.append(f'''Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}'''							)

          if sorted(set(import_dict_objects[key]							)							) != sorted(set(type_hint_objects[key]							)							):
               lowercase_					:       Dict     				=					'base imports' if key == 'none' else f'''{key} backend'''
               errors.append(f'''Differences for {name}:'''							)
               for a in type_hint_objects[key]:
                    if a not in import_dict_objects[key]:
                         errors.append(f'''  {a} in TYPE_HINT but not in _import_structure.'''							)
               for a in import_dict_objects[key]:
                    if a not in type_hint_objects[key]:
                         errors.append(f'''  {a} in _import_structure but not in TYPE_HINT.'''							)
     return errors

def _UpperCamelCase   (       ):
     lowercase_					:       List[Any]     				=					[]
     for root, _, files in os.walk(SCREAMING_SNAKE_CASE_							):
          if "__init__.py" in files:
               lowercase_					:       int     				=					os.path.join(SCREAMING_SNAKE_CASE_  ,       '__init__.py'							)
               lowercase_					:       Union[str, Any]     				=					parse_init(SCREAMING_SNAKE_CASE_							)
               if objects is not None:
                    lowercase_					:       Dict     				=					analyze_results(*SCREAMING_SNAKE_CASE_							)
                    if len(SCREAMING_SNAKE_CASE_							) > 0:
                         lowercase_					:       Optional[Any]     				=					f'''Problem in {fname}, both halves do not define the same objects.\n{errors[0]}'''
                         failures.append('\n'.join(SCREAMING_SNAKE_CASE_							)							)
     if len(SCREAMING_SNAKE_CASE_							) > 0:
          raise ValueError('\n\n'.join(SCREAMING_SNAKE_CASE_							)							)

def _UpperCamelCase   (       ):
     lowercase_					:       Optional[int]     				=					[]
     for path, directories, files in os.walk(SCREAMING_SNAKE_CASE_							):
          for folder in directories:
               # Ignore private modules
               if folder.startswith('_'							):
                    directories.remove(SCREAMING_SNAKE_CASE_							)
                    continue
               # Ignore leftovers from branches (empty folders apart from pycache)
               if len(list((Path(SCREAMING_SNAKE_CASE_							) / folder).glob('*.py'							)							)							) == 0:
                    continue
               lowercase_					:       Optional[Any]     				=					str((Path(SCREAMING_SNAKE_CASE_							) / folder).relative_to(SCREAMING_SNAKE_CASE_							)							)
               lowercase_					:       Tuple     				=					short_path.replace(os.path.sep  ,       '.'							)
               submodules.append(SCREAMING_SNAKE_CASE_							)
          for fname in files:
               if fname == "__init__.py":
                    continue
               lowercase_					:       Union[str, Any]     				=					str((Path(SCREAMING_SNAKE_CASE_							) / fname).relative_to(SCREAMING_SNAKE_CASE_							)							)
               lowercase_					:       List[str]     				=					short_path.replace('.py'  ,       ''							).replace(os.path.sep  ,       '.'							)
               if len(submodule.split('.'							)							) == 1:
                    submodules.append(SCREAMING_SNAKE_CASE_							)
     return submodules


_A								=   [
    'convert_pytorch_checkpoint_to_tf2',
    'modeling_flax_pytorch_utils',
]

def _UpperCamelCase   (       ):
     # This is to make sure the transformers module imported is the one in the repo.
     lowercase_					:       str     				=					importlib.util.spec_from_file_location(
         'transformers'  ,       os.path.join(SCREAMING_SNAKE_CASE_  ,       '__init__.py'							)  ,       submodule_search_locations=[PATH_TO_TRANSFORMERS]  ,       )
     lowercase_					:       Union[str, Any]     				=					spec.loader.load_module()

     lowercase_					:       str     				=					[
         module
         for module in get_transformers_submodules()
         if module not in IGNORE_SUBMODULES and module not in transformers._import_structure.keys()
     ]
     if len(SCREAMING_SNAKE_CASE_							) > 0:
          lowercase_					:       int     				=					'\n'.join(f'''- {module}''' for module in module_not_registered							)
          raise ValueError(
              'The following submodules are not properly registered in the main init of Transformers:\n'
              f'''{list_of_modules}\n'''
              'Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value.'							)


if __name__ == "__main__":
 check_all_inits()
 check_submodules()





'''simple docstring'''




import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_A								=   logging.get_logger(__name__)

_A								=   '▁'

_A								=   {'vocab_file': 'vocab.txt', 'sentencepiece_model_ckpt': 'sentencepiece.bpe.model'}

_A								=   {
    'sentencepiece_model_file': 'sentencepiece.bpe.model',
    'vocab_file': 'vocab.txt',
}

_A								=   {
    'vocab_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
    },
    'sentencepiece_model_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
    },
}

_A								=   {
    'ernie-m-base': 5_1_4,
    'ernie-m-large': 5_1_4,
}

_A								=   {
    'ernie-m-base': {'do_lower_case': False},
    'ernie-m-large': {'do_lower_case': False},
}


class       UpperCAmelCase__  (       _snake_case			):






   """simple docstring"""
   A							:       List[str]     =    ["input_ids"]

   A							:       Optional[Any]     =    VOCAB_FILES_NAMES
   A							:       Dict     =    PRETRAINED_INIT_CONFIGURATION
   A							:       int     =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   A							:       List[str]     =    PRETRAINED_VOCAB_FILES_MAP
   A							:       Any     =    RESOURCE_FILES_NAMES
   def __init__(self       ,				_a       ,				_a=None       ,				_a=False       ,				_a="utf8"       ,				_a="[UNK]"       ,				_a="[SEP]"       ,				_a="[PAD]"       ,				_a="[CLS]"       ,				_a="[MASK]"       ,				_a = None       ,				**_a       ,				)    ->     None:
        # Mask token behave like a normal word, i.e. include the space before it and
        # is included in the raw text, there should be a match in a non-normalized sentence.

        lowercase_					:       Optional[Any]     				=					{} if sp_model_kwargs is None else sp_model_kwargs
        super().__init__(
            do_lower_case=_a       ,				unk_token=_a       ,				sep_token=_a       ,				pad_token=_a       ,				cls_token=_a       ,				mask_token=_a       ,				vocab_file=_a       ,				encoding=_a       ,				sp_model_kwargs=self.sp_model_kwargs       ,				**_a       ,				)
        lowercase_					:       int     				=					do_lower_case
        lowercase_					:       str     				=					sentencepiece_model_ckpt
        lowercase_					:       Tuple     				=					spm.SentencePieceProcessor(**self.sp_model_kwargs				)
        self.sp_model.Load(_a				)

        # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
        if vocab_file is not None:
             lowercase_					:       Union[str, Any]     				=					self.load_vocab(filepath=_a				)
        else:
             lowercase_					:       Any     				=					{self.sp_model.id_to_piece(_a				): id for id in range(self.sp_model.get_piece_size()				)}
        lowercase_					:       Optional[int]     				=					{v: k for k, v in self.vocab.items()}
   def 		_lowerCamelCase (self       ,				_a				)    ->     Any:
        if text is None:
             return None

        lowercase_					:       Dict     				=					self.tokenize(_a				)
        lowercase_				,lowercase_					:       Dict     				=					'', []

        for i, ch in enumerate(_a				):
             if ch in self.SP_CHAR_MAPPING:
                  lowercase_					:       str     				=					self.SP_CHAR_MAPPING.get(_a				)
             else:
                  lowercase_					:       List[str]     				=					unicodedata.normalize('NFKC'       ,				_a				)
             if self.is_whitespace(_a				):
                  continue
             normalized_text += ch
             char_mapping.extend([i] * len(_a				)				)

        lowercase_				,lowercase_				,lowercase_					:       int     				=					normalized_text, [], 0

        if self.do_lower_case:
             lowercase_					:       Optional[Any]     				=					text.lower()

        for token in split_tokens:
             if token[:1] == "▁":
                  lowercase_					:       Tuple     				=					token[1:]
             lowercase_					:       List[str]     				=					text[offset:].index(_a				) + offset
             lowercase_					:       Any     				=					start + len(_a				)

             token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)				)
             lowercase_					:       Tuple     				=					end
        return token_mapping
   @property
   def 		_lowerCamelCase (self				)    ->     List[Any]:
        return len(self.vocab				)
   def 		_lowerCamelCase (self				)    ->     int:
        return dict(self.vocab       ,				**self.added_tokens_encoder				)
   def __getstate__(self				)    ->     Optional[Any]:
        lowercase_					:       Dict     				=					self.__dict__.copy()
        lowercase_					:       List[str]     				=					None
        return state
   def __setstate__(self       ,				_a				)    ->     Any:
        lowercase_					:       Optional[int]     				=					d

        # for backward compatibility
        if not hasattr(self       ,				'sp_model_kwargs'				):
             lowercase_					:       Tuple     				=					{}

        lowercase_					:       Optional[int]     				=					spm.SentencePieceProcessor(**self.sp_model_kwargs				)
        self.sp_model.Load(self.sentencepiece_model_ckpt				)
   def 		_lowerCamelCase (self       ,				_a				)    ->     str:
        return "".join((self.SP_CHAR_MAPPING.get(_a       ,				_a				) for c in text)				)
   def 		_lowerCamelCase (self       ,				_a       ,				_a=False       ,				_a=64       ,				_a=0.1				)    ->     Optional[int]:

        if self.sp_model_kwargs.get('enable_sampling'				) is True:
             lowercase_					:       Optional[int]     				=					True
        if self.sp_model_kwargs.get('alpha'				) is not None:
             lowercase_					:       Union[str, Any]     				=					self.sp_model_kwargs.get('alpha'				)
        if self.sp_model_kwargs.get('nbest_size'				) is not None:
             lowercase_					:       Optional[int]     				=					self.sp_model_kwargs.get('nbest_size'				)

        if not enable_sampling:
             lowercase_					:       Optional[Any]     				=					self.sp_model.EncodeAsPieces(_a				)
        else:
             lowercase_					:       List[Any]     				=					self.sp_model.SampleEncodeAsPieces(_a       ,				_a       ,				_a				)
        lowercase_					:       int     				=					[]
        for pi, piece in enumerate(_a				):
             if piece == SPIECE_UNDERLINE:
                  if not pieces[pi + 1].startswith(_a				) and pi != 0:
                       new_pieces.append(_a				)
                       continue
                  else:
                       continue
             lowercase_					:       Optional[Any]     				=					0
             for i, chunk in enumerate(_a				):
                  if chunk == SPIECE_UNDERLINE:
                       continue
                  if self.is_ch_char(_a				) or self.is_punct(_a				):
                       if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                            new_pieces.append(piece[lst_i:i]				)
                       new_pieces.append(_a				)
                       lowercase_					:       List[Any]     				=					i + 1
                  elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
                       if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                            new_pieces.append(piece[lst_i:i]				)
                       lowercase_					:       Optional[Any]     				=					i
                  elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
                       if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                            new_pieces.append(piece[lst_i:i]				)
                       lowercase_					:       int     				=					i
             if len(_a				) > lst_i:
                  new_pieces.append(piece[lst_i:]				)
        return new_pieces
   def 		_lowerCamelCase (self       ,				_a				)    ->     Union[str, Any]:
        lowercase_					:       Optional[Any]     				=					''.join(_a				).replace(_a       ,				' '				).strip()
        return out_string
   def 		_lowerCamelCase (self       ,				_a				)    ->     Optional[int]:
        lowercase_					:       Optional[Any]     				=					self.convert_ids_to_tokens(_a				)
        lowercase_					:       Tuple     				=					''.join(_a				).replace(_a       ,				' '				).strip()
        return out_string
   def 		_lowerCamelCase (self       ,				_a				)    ->     Optional[int]:
        return self.vocab.get(_a       ,				self.vocab.get(self.unk_token				)				)
   def 		_lowerCamelCase (self       ,				_a				)    ->     int:
        return self.reverse_vocab.get(_a       ,				self.unk_token				)
   def 		_lowerCamelCase (self       ,				_a       ,				_a=None				)    ->     Any:
        if token_ids_a is None:
             return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
        lowercase_					:       Tuple     				=					[self.cls_token_id]
        lowercase_					:       int     				=					[self.sep_token_id]
        return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep
   def 		_lowerCamelCase (self       ,				_a       ,				_a=None				)    ->     Optional[Any]:
        if offset_mapping_a is None:
             return [(0, 0)] + offset_mapping_a + [(0, 0)]

        return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]
   def 		_lowerCamelCase (self       ,				_a       ,				_a=None       ,				_a=False				)    ->     Union[str, Any]:

        if already_has_special_tokens:
             if token_ids_a is not None:
                  raise ValueError(
                      'You should not supply a second sequence if the provided sequence of '
                      'ids is already formatted with special tokens for the model.'				)
             return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

        if token_ids_a is not None:
             return [1] + ([0] * len(_a				)) + [1, 1] + ([0] * len(_a				)) + [1]
        return [1] + ([0] * len(_a				)) + [1]
   def 		_lowerCamelCase (self       ,				_a       ,				_a = None				)    ->     List[int]:
        # called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
        if token_ids_a is None:
             # [CLS] X [SEP]
             return (len(_a				) + 2) * [0]

        # [CLS] A [SEP] [SEP] B [SEP]
        return [0] * (len(_a				) + 1) + [1] * (len(_a				) + 3)
   def 		_lowerCamelCase (self       ,				_a				)    ->     int:
        if "\u4e00" <= char <= "\u9fff":
             return True
        return False
   def 		_lowerCamelCase (self       ,				_a				)    ->     List[str]:
        if ("a" <= char <= "z") or ("A" <= char <= "Z"):
             return True
        return False
   def 		_lowerCamelCase (self       ,				_a				)    ->     int:
        if char in ",;:.?!~，；：。？！《》【】":
             return True
        return False
   def 		_lowerCamelCase (self       ,				_a				)    ->     List[Any]:
        if char == " " or char == "\t" or char == "\n" or char == "\r":
             return True
        if len(_a				) == 1:
             lowercase_					:       Optional[Any]     				=					unicodedata.category(_a				)
             if cat == "Zs":
                  return True
        return False
   def 		_lowerCamelCase (self       ,				_a				)    ->     Tuple:
        lowercase_					:       List[Any]     				=					{}
        with io.open(_a       ,				'r'       ,				encoding='utf-8'				) as f:
             for index, line in enumerate(_a				):
                  lowercase_					:       Dict     				=					line.rstrip('\n'				)
                  lowercase_					:       Dict     				=					int(_a				)

        return token_to_idx





   def 		_lowerCamelCase (self       ,				_a       ,				_a = None				)    ->     Tuple[str]:
        lowercase_					:       Tuple     				=					0
        if os.path.isdir(_a				):
             lowercase_					:       Any     				=					os.path.join(
                 _a       ,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']				)
        else:
             lowercase_					:       Tuple     				=					(filename_prefix + '-' if filename_prefix else '') + save_directory
        with open(_a       ,				'w'       ,				encoding='utf-8'				) as writer:
             for token, token_index in sorted(self.vocab.items()       ,				key=lambda _a				: kv[1]				):
                  if index != token_index:
                       logger.warning(
                           f'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
                           ' Please check that the vocabulary is not corrupted!'				)
                       lowercase_					:       Tuple     				=					token_index
                  writer.write(token + '\n'				)
                  index += 1

        lowercase_					:       List[str]     				=					os.path.join(_a       ,				'sentencepiece.bpe.model'				)
        with open(_a       ,				'wb'				) as fi:
             lowercase_					:       Tuple     				=					self.sp_model.serialized_model_proto()
             fi.write(_a				)

        return (vocab_file,)




