"""simple docstring"""







import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
       import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__SCREAMING_SNAKE_CASE								=logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE								={"vocab_file": "vocab.txt"}

__SCREAMING_SNAKE_CASE								={
    "vocab_file": {
        "openbmb/cpm-ant-10b": "https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt",
    },
}

__SCREAMING_SNAKE_CASE								={
    "openbmb/cpm-ant-10b": 1024,
}



def     lowercase__( __SCREAMING_SNAKE_CASE							:  Tuple   ):
     lowercase_			:       List[Any]									=			collections.OrderedDict()
     with open(__SCREAMING_SNAKE_CASE  ,    'r'  ,    encoding='utf-8'   ) as reader:
          lowercase_			:       Optional[Any]									=			reader.readlines()
     for index, token in enumerate(__SCREAMING_SNAKE_CASE   ):
          lowercase_			:       int									=			token.rstrip('\n'   )
          lowercase_			:       str									=			index
     return vocab







class  UpperCamelCase	(    __a						):
   def __init__(    self  ,__UpperCamelCase  ,__UpperCamelCase="<unk>"  ,__UpperCamelCase=200    )			->   Any:


        '''simple docstring'''


        lowercase_			:       Union[str, Any]									=			vocab
        lowercase_			:       str									=			unk_token
        lowercase_			:       int									=			max_input_chars_per_word







   def     _UpperCAmelCase     (    self  ,__UpperCamelCase    )			->   str:


        '''simple docstring'''


        lowercase_			:       Optional[Any]									=			list(A__    )
        if len(A__    ) > self.max_input_chars_per_word:
             return [self.unk_token]

        lowercase_			:       str									=			0
        lowercase_			:       Dict									=			[]
        while start < len(A__    ):
             lowercase_			:       Union[str, Any]									=			len(A__    )
             lowercase_			:       Dict									=			None
             while start < end:
                  lowercase_			:       Union[str, Any]									=			''.join(chars[start:end]    )
                  if substr in self.vocab:
                       lowercase_			:       Any									=			substr
                       break
                  end -= 1
             if cur_substr is None:
                  sub_tokens.append(self.unk_token    )
                  start += 1
             else:
                  sub_tokens.append(A__    )
                  lowercase_			:       str									=			end

        return sub_tokens







class  UpperCamelCase	(    __a						):
   lowercase							=						VOCAB_FILES_NAMES
   lowercase							=						PRETRAINED_VOCAB_FILES_MAP
   lowercase							=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   lowercase							=						['input_ids', 'attention_mask']
   lowercase							=						False
   def __init__(    self  ,__UpperCamelCase  ,__UpperCamelCase="<d>"  ,__UpperCamelCase="</d>"  ,__UpperCamelCase="<s>"  ,__UpperCamelCase="</s>"  ,__UpperCamelCase="<pad>"  ,__UpperCamelCase="<unk>"  ,__UpperCamelCase="</n>"  ,__UpperCamelCase="</_>"  ,__UpperCamelCase="left"  ,**__UpperCamelCase  ,)			->   Dict:


        '''simple docstring'''


        requires_backends(self  ,['jieba']    )
        super().__init__(
            bod_token=A__  ,eod_token=A__  ,bos_token=A__  ,eos_token=A__  ,pad_token=A__  ,unk_token=A__  ,line_token=A__  ,space_token=A__  ,padding_side=A__  ,**A__  ,)
        lowercase_			:       Dict									=			bod_token
        lowercase_			:       Dict									=			eod_token
        lowercase_			:       int									=			load_vocab(A__    )
        lowercase_			:       List[str]									=			self.encoder[space_token]
        lowercase_			:       Optional[int]									=			self.encoder[line_token]

        del self.encoder[space_token]
        del self.encoder[line_token]

        lowercase_			:       Optional[int]									=			collections.OrderedDict(sorted(self.encoder.items()  ,key=lambda __UpperCamelCase    : x[1]    )    )
        lowercase_			:       Union[str, Any]									=			{v: k for k, v in self.encoder.items()}

        lowercase_			:       Any									=			WordpieceTokenizer(vocab=self.encoder  ,unk_token=self.unk_token    )
   @property
   def     _UpperCAmelCase     (    self    )			->   Any:


        '''simple docstring'''


        return self.encoder[self.bod_token]
   @property
   def     _UpperCAmelCase     (    self    )			->   int:


        '''simple docstring'''


        return self.encoder[self.eod_token]
   @property
   def     _UpperCAmelCase     (    self    )			->   List[str]:


        '''simple docstring'''


        return self.encoder["\n"]
   @property
   def     _UpperCAmelCase     (    self    )			->   int:


        '''simple docstring'''


        return len(self.encoder    )
   def     _UpperCAmelCase     (    self    )			->   int:


        '''simple docstring'''


        return dict(self.encoder  ,**self.added_tokens_encoder    )
   def     _UpperCAmelCase     (    self  ,__UpperCamelCase    )			->   List[Any]:


        '''simple docstring'''


        lowercase_			:       str									=			[]
        for x in jieba.cut(A__  ,cut_all=A__    ):
             output_tokens.extend(self.wordpiece_tokenizer.tokenize(A__    )    )
        return output_tokens
   def     _UpperCAmelCase     (    self  ,__UpperCamelCase  ,**__UpperCamelCase    )			->   Optional[Any]:


        '''simple docstring'''


        lowercase_			:       Dict									=			[i for i in token_ids if i >= 0]
        lowercase_			:       Union[str, Any]									=			[
            x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
        ]
        return super()._decode(A__  ,**A__    )
   def     _UpperCAmelCase     (    self  ,__UpperCamelCase    )			->   Dict:


        '''simple docstring'''


        return token in self.encoder
   def     _UpperCAmelCase     (    self  ,__UpperCamelCase    )			->   str:


        '''simple docstring'''


        return "".join(A__    )
   def     _UpperCAmelCase     (    self  ,__UpperCamelCase    )			->   int:


        '''simple docstring'''


        return self.encoder.get(A__  ,self.encoder.get(self.unk_token    )    )
   def     _UpperCAmelCase     (    self  ,__UpperCamelCase    )			->   int:


        '''simple docstring'''


        return self.decoder.get(A__  ,self.unk_token    )
   def     _UpperCAmelCase     (    self  ,__UpperCamelCase  ,__UpperCamelCase = None    )			->   Tuple[str]:


        '''simple docstring'''


        if os.path.isdir(A__    ):
             lowercase_			:       Tuple									=			os.path.join(
                 A__  ,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )
        else:
             lowercase_			:       str									=			(filename_prefix + '-' if filename_prefix else '') + save_directory
        lowercase_			:       int									=			0
        if " " in self.encoder:
             lowercase_			:       Optional[int]									=			self.encoder[' ']
             del self.encoder[" "]
        if "\n" in self.encoder:
             lowercase_			:       str									=			self.encoder['\n']
             del self.encoder["\n"]
        lowercase_			:       int									=			collections.OrderedDict(sorted(self.encoder.items()  ,key=lambda __UpperCamelCase    : x[1]    )    )
        with open(A__  ,'w'  ,encoding='utf-8'    ) as writer:
             for token, token_index in self.encoder.items():
                  if index != token_index:
                       logger.warning(
                           f'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
                           ' Please check that the vocabulary is not corrupted!'    )
                       lowercase_			:       List[Any]									=			token_index
                  writer.write(token + '\n'    )
                  index += 1
        return (vocab_file,)
   def     _UpperCAmelCase     (    self  ,__UpperCamelCase  ,__UpperCamelCase = None    )			->   List[int]:


        '''simple docstring'''


        if token_ids_a is None:
             return [self.bos_token_id] + token_ids_a
        return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a







   def     _UpperCAmelCase     (    self  ,__UpperCamelCase  ,__UpperCamelCase = None  ,__UpperCamelCase = False    )			->   List[int]:


        '''simple docstring'''


        if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=A__  ,token_ids_a=A__  ,already_has_special_tokens=A__    )

        if token_ids_a is not None:
             return [1] + ([0] * len(A__    )) + [1] + ([0] * len(A__    ))
        return [1] + ([0] * len(A__    ))





"""simple docstring"""







import logging
import os
from logging import (
    CRITICAL,  # NOQA
    DEBUG,  # NOQA
    ERROR,  # NOQA
    FATAL,  # NOQA
    INFO,  # NOQA
    NOTSET,  # NOQA
    WARN,  # NOQA
    WARNING,  # NOQA
)
from typing import Optional

from tqdm import auto as tqdm_lib


__SCREAMING_SNAKE_CASE								={
    "debug": logging.DEBUG,
    "info": logging.INFO,
    "warning": logging.WARNING,
    "error": logging.ERROR,
    "critical": logging.CRITICAL,
}

__SCREAMING_SNAKE_CASE								=logging.WARNING



def     lowercase__( ):
     lowercase_			:       List[Any]									=			os.getenv('DATASETS_VERBOSITY'  ,    __SCREAMING_SNAKE_CASE   )
     if env_level_str:
          if env_level_str in log_levels:
               return log_levels[env_level_str]
          else:
               logging.getLogger().warning(
                   F'''Unknown option DATASETS_VERBOSITY={env_level_str}, '''
                   F'''has to be one of: { ", ".join(log_levels.keys()   ) }'''   )
     return _default_log_level



def     lowercase__( ):
     return __name__.split('.'   )[0]



def     lowercase__( ):
     return logging.getLogger(_get_library_name()   )



def     lowercase__( ):
     # Apply our default configuration to the library root logger.
     lowercase_			:       Optional[int]									=			_get_library_root_logger()
     library_root_logger.setLevel(_get_default_logging_level()   )



def     lowercase__( ):
     lowercase_			:       int									=			_get_library_root_logger()
     library_root_logger.setLevel(logging.NOTSET   )



def     lowercase__( __SCREAMING_SNAKE_CASE							:  Optional[str] = None   ):
     if name is None:
          lowercase_			:       Union[str, Any]									=			_get_library_name()
     return logging.getLogger(__SCREAMING_SNAKE_CASE   )



def     lowercase__( ):
     return _get_library_root_logger().getEffectiveLevel()



def     lowercase__( __SCREAMING_SNAKE_CASE							:  int   ):
     _get_library_root_logger().setLevel(__SCREAMING_SNAKE_CASE   )



def     lowercase__( ):
     return set_verbosity(__SCREAMING_SNAKE_CASE   )



def     lowercase__( ):
     return set_verbosity(__SCREAMING_SNAKE_CASE   )



def     lowercase__( ):
     return set_verbosity(__SCREAMING_SNAKE_CASE   )



def     lowercase__( ):
     return set_verbosity(__SCREAMING_SNAKE_CASE   )



def     lowercase__( ):
     lowercase_			:       Tuple									=			False



def     lowercase__( ):
     lowercase_			:       Any									=			True


# Configure the library root logger at the module level (singleton-like)
_configure_library_root_logger()







class  UpperCamelCase	:
   def __init__(    self  ,*__UpperCamelCase  ,**__UpperCamelCase    )			->   int:  # pylint: disable=unused-argument


        '''simple docstring'''


        lowercase_			:       Optional[Any]									=			args[0] if args else None
   def __iter__(    self    )			->   Union[str, Any]:


        '''simple docstring'''


        return iter(self._iterator    )
   def __getattr__(    self  ,__UpperCamelCase    )			->   List[Any]:


        '''simple docstring'''


        def empty_fn(*__UpperCamelCase  ,**__UpperCamelCase    ):  # pylint: disable=unused-argument
             return

        return empty_fn
   def __enter__(    self    )			->   Optional[int]:


        '''simple docstring'''


        return self







   def __exit__(    self  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase    )			->   Dict:


        '''simple docstring'''


        return






__SCREAMING_SNAKE_CASE								=True







class  UpperCamelCase	:
   def __call__(    self  ,*__UpperCamelCase  ,__UpperCamelCase=False  ,**__UpperCamelCase    )			->   Tuple:


        '''simple docstring'''


        if _tqdm_active and not disable:
             return tqdm_lib.tqdm(*__UpperCamelCase  ,**__UpperCamelCase    )
        else:
             return EmptyTqdm(*__UpperCamelCase  ,**__UpperCamelCase    )
   def     _UpperCAmelCase     (    self  ,*__UpperCamelCase  ,**__UpperCamelCase    )			->   str:


        '''simple docstring'''


        lowercase_			:       Tuple									=			None
        if _tqdm_active:
             return tqdm_lib.tqdm.set_lock(*__UpperCamelCase  ,**__UpperCamelCase    )







   def     _UpperCAmelCase     (    self    )			->   Any:


        '''simple docstring'''


        if _tqdm_active:
             return tqdm_lib.tqdm.get_lock()






__SCREAMING_SNAKE_CASE								=_tqdm_cls()



def     lowercase__( ):
     global _tqdm_active
     return bool(_tqdm_active   )



def     lowercase__( ):
     global _tqdm_active
     lowercase_			:       Union[str, Any]									=			True







def     lowercase__( ):
     global _tqdm_active
     lowercase_			:       Optional[int]									=			False

