





'''simple docstring'''







import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
   import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_lowercase       						=			logging.get_logger(__name__)

_lowercase       						=			{'vocab_file': 'vocab.txt'}

_lowercase       						=			{
    'vocab_file': {
        'openbmb/cpm-ant-10b': 'https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt',
    },
}

_lowercase       						=			{
    'openbmb/cpm-ant-10b': 1_024,
}

def    __UpperCamelCase	(	a    : Tuple     ) ->List[Any]:
 snake_case				=			collections.OrderedDict()
 with open(a					,   '''r'''					,   encoding='''utf-8'''     ) as reader:
  snake_case				=			reader.readlines()
 for index, token in enumerate(a     ):
  snake_case				=			token.rstrip('''\n'''     )
  snake_case				=			index
 return vocab
class 			_lowercase						(       __a					):







     def __init__( self    ,   A__    ,   A__="<unk>"    ,   A__=2_00    ) ->	Any:
      snake_case				=			vocab
      snake_case				=			unk_token
      snake_case				=			max_input_chars_per_word







     def     UpperCamelCase			( self    ,   A__    ) ->	str:
      snake_case				=			list(A__    )
      if len(A__    ) > self.max_input_chars_per_word:
       return [self.unk_token]

      snake_case				=			0
      snake_case				=			[]
      while start < len(A__    ):
       snake_case				=			len(A__    )
       snake_case				=			None
       while start < end:
        snake_case				=			''''''.join(chars[start:end]    )
        if substr in self.vocab:
         snake_case				=			substr
         break
        end -= 1
       if cur_substr is None:
        sub_tokens.append(self.unk_token    )
        start += 1
       else:
        sub_tokens.append(A__    )
        snake_case				=			end

      return sub_tokens




class 			_lowercase						(       __a					):
     _UpperCAmelCase										=					VOCAB_FILES_NAMES
     _UpperCAmelCase										=					PRETRAINED_VOCAB_FILES_MAP
     _UpperCAmelCase										=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _UpperCAmelCase										=					['''input_ids''', '''attention_mask''']
     _UpperCAmelCase										=					False







     def __init__( self    ,   A__    ,   A__="<d>"    ,   A__="</d>"    ,   A__="<s>"    ,   A__="</s>"    ,   A__="<pad>"    ,   A__="<unk>"    ,   A__="</n>"    ,   A__="</_>"    ,   A__="left"    ,   **A__    ,   ) ->	Dict:
      requires_backends(self    ,   ['''jieba''']    )
      super().__init__(
          bod_token=A__    ,   eod_token=A__    ,   bos_token=A__    ,   eos_token=A__    ,   pad_token=A__    ,   unk_token=A__    ,   line_token=A__    ,   space_token=A__    ,   padding_side=A__    ,   **A__    ,   )
      snake_case				=			bod_token
      snake_case				=			eod_token
      snake_case				=			load_vocab(A__    )
      snake_case				=			self.encoder[space_token]
      snake_case				=			self.encoder[line_token]

      del self.encoder[space_token]
      del self.encoder[line_token]

      snake_case				=			collections.OrderedDict(sorted(self.encoder.items()    ,   key=lambda A__    : x[1]    )    )
      snake_case				=			{v: k for k, v in self.encoder.items()}

      snake_case				=			WordpieceTokenizer(vocab=self.encoder    ,   unk_token=self.unk_token    )







     @property
     def     UpperCamelCase			( self    ) ->	Any:
      return self.encoder[self.bod_token]







     @property
     def     UpperCamelCase			( self    ) ->	int:
      return self.encoder[self.eod_token]







     @property
     def     UpperCamelCase			( self    ) ->	List[str]:
      return self.encoder["\n"]







     @property
     def     UpperCamelCase			( self    ) ->	int:
      return len(self.encoder    )







     def     UpperCamelCase			( self    ) ->	int:
      return dict(self.encoder    ,   **self.added_tokens_encoder    )







     def     UpperCamelCase			( self    ,   A__    ) ->	List[Any]:
      snake_case				=			[]
      for x in jieba.cut(A__    ,   cut_all=A__    ):
       output_tokens.extend(self.wordpiece_tokenizer.tokenize(A__    )    )
      return output_tokens







     def     UpperCamelCase			( self    ,   A__    ,   **A__    ) ->	Optional[Any]:
      snake_case				=			[i for i in token_ids if i >= 0]
      snake_case				=			[
          x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
      ]
      return super()._decode(A__    ,   **A__    )







     def     UpperCamelCase			( self    ,   A__    ) ->	Dict:
      return token in self.encoder







     def     UpperCamelCase			( self    ,   A__    ) ->	str:
      return "".join(A__    )







     def     UpperCamelCase			( self    ,   A__    ) ->	int:
      return self.encoder.get(A__    ,   self.encoder.get(self.unk_token    )    )







     def     UpperCamelCase			( self    ,   A__    ) ->	int:
      return self.decoder.get(A__    ,   self.unk_token    )







     def     UpperCamelCase			( self    ,   A__    ,   A__ = None    ) ->	Tuple[str]:
      if os.path.isdir(A__    ):
       snake_case				=			os.path.join(
           A__    ,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']    )
      else:
       snake_case				=			(filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
      snake_case				=			0
      if " " in self.encoder:
       snake_case				=			self.encoder[''' ''']
       del self.encoder[" "]
      if "\n" in self.encoder:
       snake_case				=			self.encoder['''\n''']
       del self.encoder["\n"]
      snake_case				=			collections.OrderedDict(sorted(self.encoder.items()    ,   key=lambda A__    : x[1]    )    )
      with open(A__    ,   '''w'''    ,   encoding='''utf-8'''    ) as writer:
       for token, token_index in self.encoder.items():
        if index != token_index:
         logger.warning(
             F"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
             ''' Please check that the vocabulary is not corrupted!'''    )
         snake_case				=			token_index
        writer.write(token + '''\n'''    )
        index += 1
      return (vocab_file,)







     def     UpperCamelCase			( self    ,   A__    ,   A__ = None    ) ->	List[int]:
      if token_ids_a is None:
       return [self.bos_token_id] + token_ids_a
      return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a







     def     UpperCamelCase			( self    ,   A__    ,   A__ = None    ,   A__ = False    ) ->	List[int]:

      if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=A__    ,   token_ids_a=A__    ,   already_has_special_tokens=A__    )

      if token_ids_a is not None:
       return [1] + ([0] * len(A__    )) + [1] + ([0] * len(A__    ))
      return [1] + ([0] * len(A__    ))








'''simple docstring'''







import inspect
import re
from hashlib import shaaaa
from typing import Dict, List

from .arrow import arrow
from .audiofolder import audiofolder
from .csv import csv
from .imagefolder import imagefolder
from .json import json
from .pandas import pandas
from .parquet import parquet
from .sql import sql  # noqa F401
from .text import text

def    __UpperCamelCase	(	a    : List[str]     ) ->str:
 snake_case				=			[]
 for line in lines:
  snake_case				=			re.sub(R'''#.*'''					,   ''''''					,   a     )  # remove comments
  if line:
   filtered_lines.append(a     )
 snake_case				=			'''\n'''.join(a     )

 # Make a hash from all this code
 snake_case				=			full_str.encode('''utf-8'''     )
 return shaaaa(a     ).hexdigest()


# get importable module names and hash for caching
_lowercase       						=			{
    'csv': (csv.__name__, _hash_python_lines(inspect.getsource(csv).splitlines())),
    'json': (json.__name__, _hash_python_lines(inspect.getsource(json).splitlines())),
    'pandas': (pandas.__name__, _hash_python_lines(inspect.getsource(pandas).splitlines())),
    'parquet': (parquet.__name__, _hash_python_lines(inspect.getsource(parquet).splitlines())),
    'arrow': (arrow.__name__, _hash_python_lines(inspect.getsource(arrow).splitlines())),
    'text': (text.__name__, _hash_python_lines(inspect.getsource(text).splitlines())),
    'imagefolder': (imagefolder.__name__, _hash_python_lines(inspect.getsource(imagefolder).splitlines())),
    'audiofolder': (audiofolder.__name__, _hash_python_lines(inspect.getsource(audiofolder).splitlines())),
}

# Used to infer the module to use based on the data files extensions
_lowercase       						=			{
    '.csv': ('csv', {}),
    '.tsv': ('csv', {'sep': '\t'}),
    '.json': ('json', {}),
    '.jsonl': ('json', {}),
    '.parquet': ('parquet', {}),
    '.arrow': ('arrow', {}),
    '.txt': ('text', {}),
}
_EXTENSION_TO_MODULE.update({ext: ('imagefolder', {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext.upper(): ('imagefolder', {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext: ('audiofolder', {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext.upper(): ('audiofolder', {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
_lowercase       						=			{'imagefolder', 'audiofolder'}

# Used to filter data files based on extensions given a module name
_lowercase       						=			{}
for _ext, (_module, _) in _EXTENSION_TO_MODULE.items():
   _MODULE_TO_EXTENSIONS.setdefault(_module, []).append(_ext)

_MODULE_TO_EXTENSIONS["imagefolder"].append('.zip')






_MODULE_TO_EXTENSIONS["audiofolder"].append('.zip')

