






"""simple docstring"""



import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


A_    =							logging.get_logger(__name__)

A_    =							'''▁'''

A_    =							{'''vocab_file''': '''vocab.txt''', '''sentencepiece_model_ckpt''': '''sentencepiece.bpe.model'''}

A_    =							{
    '''sentencepiece_model_file''': '''sentencepiece.bpe.model''',
    '''vocab_file''': '''vocab.txt''',
}

A_    =							{
    '''vocab_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
    },
    '''sentencepiece_model_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
    },
}

A_    =							{
    '''ernie-m-base''': 514,
    '''ernie-m-large''': 514,
}

A_    =							{
    '''ernie-m-base''': {'''do_lower_case''': False},
    '''ernie-m-large''': {'''do_lower_case''': False},
}






class 							__SCREAMING_SNAKE_CASE						(				UpperCamelCase		):
							snake_case_     =							["input_ids"]

							snake_case_     =							VOCAB_FILES_NAMES
							snake_case_     =							PRETRAINED_INIT_CONFIGURATION
							snake_case_     =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							snake_case_     =							PRETRAINED_VOCAB_FILES_MAP
							snake_case_     =							RESOURCE_FILES_NAMES


							def __init__(			self      :						str			,    snake_case      :						Optional[Any]			,    snake_case      :						Dict=None			,    snake_case      :						List[Any]=False			,    snake_case      :						Any="utf8"			,    snake_case      :						List[str]="[UNK]"			,    snake_case      :						List[Any]="[SEP]"			,    snake_case      :						List[Any]="[PAD]"			,    snake_case      :						Union[str, Any]="[CLS]"			,    snake_case      :						Optional[Any]="[MASK]"			,    snake_case      :						Optional[Dict[str, Any]] = None			,    **snake_case      :						Union[str, Any]			,    ):
													'''simple docstring'''





													A__						:  Optional[Any]		  =  {} if sp_model_kwargs is None else sp_model_kwargs
													super().__init__(
													    do_lower_case=snake_case			,    unk_token=snake_case			,    sep_token=snake_case			,    pad_token=snake_case			,    cls_token=snake_case			,    mask_token=snake_case			,    vocab_file=snake_case			,    encoding=snake_case			,    sp_model_kwargs=self.sp_model_kwargs			,    **snake_case			,    )
													A__						:  Optional[int]		  =  do_lower_case
													A__						:  int		  =  sentencepiece_model_ckpt
													A__						:  Optional[Any]		  =  spm.SentencePieceProcessor(**self.sp_model_kwargs							)
													self.sp_model.Load(snake_case							)

													# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
													if vocab_file is not None:
																			A__						:  Union[str, Any]		  =  self.load_vocab(filepath=snake_case							)
													else:
																			A__						:  int		  =  {self.sp_model.id_to_piece(snake_case							): id for id in range(self.sp_model.get_piece_size()							)}
													A__						:  Dict		  =  {v: k for k, v in self.vocab.items()}


							def 						_UpperCamelCase       (			self      :						str			,    snake_case      :						List[Any]							):
													'''simple docstring'''





													if text is None:
																			return None

													A__						:  List[str]		  =  self.tokenize(snake_case							)
													A__  ,  A__						:  Optional[Any]		  =  """""", []

													for i, ch in enumerate(snake_case							):
																			if ch in self.SP_CHAR_MAPPING:
																									A__						:  Dict		  =  self.SP_CHAR_MAPPING.get(snake_case							)
																			else:
																									A__						:  Dict		  =  unicodedata.normalize("""NFKC"""			,    snake_case							)
																			if self.is_whitespace(snake_case							):
																									continue
																			normalized_text += ch
																			char_mapping.extend([i] * len(snake_case							)							)

													A__  ,  A__  ,  A__						:  str		  =  normalized_text, [], 0

													if self.do_lower_case:
																			A__						:  Union[str, Any]		  =  text.lower()

													for token in split_tokens:
																			if token[:1] == "▁":
																									A__						:  Any		  =  token[1:]
																			A__						:  Union[str, Any]		  =  text[offset:].index(snake_case							) + offset
																			A__						:  Union[str, Any]		  =  start + len(snake_case							)

																			token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)							)
																			A__						:  int		  =  end
													return token_mapping


							@property
							def 						_UpperCamelCase       (			self      :						Any							):
													'''simple docstring'''





													return len(self.vocab							)


							def 						_UpperCamelCase       (			self      :						str							):
													'''simple docstring'''





													return dict(self.vocab			,    **self.added_tokens_encoder							)


							def __getstate__(			self      :						Union[str, Any]							):
													'''simple docstring'''





													A__						:  Dict		  =  self.__dict__.copy()
													A__						:  Any		  =  None
													return state


							def __setstate__(			self      :						str			,    snake_case      :						Tuple							):
													'''simple docstring'''





													A__						:  Optional[int]		  =  d

													# for backward compatibility
													if not hasattr(self			,    """sp_model_kwargs"""							):
																			A__						:  Dict		  =  {}

													A__						:  Optional[int]		  =  spm.SentencePieceProcessor(**self.sp_model_kwargs							)
													self.sp_model.Load(self.sentencepiece_model_ckpt							)


							def 						_UpperCamelCase       (			self      :						Tuple			,    snake_case      :						Union[str, Any]							):
													'''simple docstring'''





													return "".join((self.SP_CHAR_MAPPING.get(snake_case			,    snake_case							) for c in text)							)


							def 						_UpperCamelCase       (			self      :						int			,    snake_case      :						Tuple			,    snake_case      :						Tuple=False			,    snake_case      :						str=64			,    snake_case      :						Tuple=0.1							):
													'''simple docstring'''





													if self.sp_model_kwargs.get("""enable_sampling"""							) is True:
																			A__						:  int		  =  True
													if self.sp_model_kwargs.get("""alpha"""							) is not None:
																			A__						:  Tuple		  =  self.sp_model_kwargs.get("""alpha"""							)
													if self.sp_model_kwargs.get("""nbest_size"""							) is not None:
																			A__						:  Dict		  =  self.sp_model_kwargs.get("""nbest_size"""							)

													if not enable_sampling:
																			A__						:  Dict		  =  self.sp_model.EncodeAsPieces(snake_case							)
													else:
																			A__						:  Dict		  =  self.sp_model.SampleEncodeAsPieces(snake_case			,    snake_case			,    snake_case							)
													A__						:  List[Any]		  =  []
													for pi, piece in enumerate(snake_case							):
																			if piece == SPIECE_UNDERLINE:
																									if not pieces[pi + 1].startswith(snake_case							) and pi != 0:
																															new_pieces.append(snake_case							)
																															continue
																									else:
																															continue
																			A__						:  Dict		  =  0
																			for i, chunk in enumerate(snake_case							):
																									if chunk == SPIECE_UNDERLINE:
																															continue
																									if self.is_ch_char(snake_case							) or self.is_punct(snake_case							):
																															if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																					new_pieces.append(piece[lst_i:i]							)
																															new_pieces.append(snake_case							)
																															A__						:  List[Any]		  =  i + 1
																									elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
																															if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																					new_pieces.append(piece[lst_i:i]							)
																															A__						:  Optional[int]		  =  i
																									elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
																															if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																					new_pieces.append(piece[lst_i:i]							)
																															A__						:  str		  =  i
																			if len(snake_case							) > lst_i:
																									new_pieces.append(piece[lst_i:]							)
													return new_pieces


							def 						_UpperCamelCase       (			self      :						List[str]			,    snake_case      :						Any							):
													'''simple docstring'''





													A__						:  Dict		  =  """""".join(snake_case							).replace(snake_case			,    """ """							).strip()
													return out_string


							def 						_UpperCamelCase       (			self      :						str			,    snake_case      :						Union[str, Any]							):
													'''simple docstring'''





													A__						:  Optional[Any]		  =  self.convert_ids_to_tokens(snake_case							)
													A__						:  int		  =  """""".join(snake_case							).replace(snake_case			,    """ """							).strip()
													return out_string


							def 						_UpperCamelCase       (			self      :						int			,    snake_case      :						Dict							):
													'''simple docstring'''





													return self.vocab.get(snake_case			,    self.vocab.get(self.unk_token							)							)


							def 						_UpperCamelCase       (			self      :						Optional[int]			,    snake_case      :						List[Any]							):
													'''simple docstring'''





													return self.reverse_vocab.get(snake_case			,    self.unk_token							)


							def 						_UpperCamelCase       (			self      :						List[str]			,    snake_case      :						Union[str, Any]			,    snake_case      :						str=None							):
													'''simple docstring'''





													if token_ids_a is None:
																			return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
													A__						:  Dict		  =  [self.cls_token_id]
													A__						:  int		  =  [self.sep_token_id]
													return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep


							def 						_UpperCamelCase       (			self      :						Tuple			,    snake_case      :						Optional[int]			,    snake_case      :						Any=None							):
													'''simple docstring'''





													if offset_mapping_a is None:
																			return [(0, 0)] + offset_mapping_a + [(0, 0)]

													return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]


							def 						_UpperCamelCase       (			self      :						Union[str, Any]			,    snake_case      :						int			,    snake_case      :						Any=None			,    snake_case      :						Tuple=False							):
													'''simple docstring'''





													if already_has_special_tokens:
																			if token_ids_a is not None:
																									raise ValueError(
																									    """You should not supply a second sequence if the provided sequence of """
																									    """ids is already formatted with special tokens for the model."""							)
																			return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

													if token_ids_a is not None:
																			return [1] + ([0] * len(snake_case							)) + [1, 1] + ([0] * len(snake_case							)) + [1]
													return [1] + ([0] * len(snake_case							)) + [1]


							def 						_UpperCamelCase       (			self      :						Optional[int]			,    snake_case      :						List[int]			,    snake_case      :						Optional[List[int]] = None							):
													'''simple docstring'''





													if token_ids_a is None:
																			# [CLS] X [SEP]
																			return (len(snake_case							) + 2) * [0]

													# [CLS] A [SEP] [SEP] B [SEP]
													return [0] * (len(snake_case							) + 1) + [1] * (len(snake_case							) + 3)


							def 						_UpperCamelCase       (			self      :						Union[str, Any]			,    snake_case      :						int							):
													'''simple docstring'''





													if "\u4e00" <= char <= "\u9fff":
																			return True
													return False


							def 						_UpperCamelCase       (			self      :						Dict			,    snake_case      :						Optional[int]							):
													'''simple docstring'''





													if ("a" <= char <= "z") or ("A" <= char <= "Z"):
																			return True
													return False


							def 						_UpperCamelCase       (			self      :						str			,    snake_case      :						int							):
													'''simple docstring'''





													if char in ",;:.?!~，；：。？！《》【】":
																			return True
													return False


							def 						_UpperCamelCase       (			self      :						Tuple			,    snake_case      :						List[str]							):
													'''simple docstring'''





													if char == " " or char == "\t" or char == "\n" or char == "\r":
																			return True
													if len(snake_case							) == 1:
																			A__						:  Tuple		  =  unicodedata.category(snake_case							)
																			if cat == "Zs":
																									return True
													return False


							def 						_UpperCamelCase       (			self      :						str			,    snake_case      :						Optional[int]							):
													'''simple docstring'''





													A__						:  str		  =  {}
													with io.open(snake_case			,    """r"""			,    encoding="""utf-8"""							) as f:
																			for index, line in enumerate(snake_case							):
																									A__						:  int		  =  line.rstrip("""\n"""							)
																									A__						:  Dict		  =  int(snake_case							)

													return token_to_idx


							def 						_UpperCamelCase       (			self      :						Any			,    snake_case      :						str			,    snake_case      :						Optional[str] = None							):
													'''simple docstring'''





													A__						:  Dict		  =  0
													if os.path.isdir(snake_case							):
																			A__						:  Any		  =  os.path.join(
																			    snake_case			,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)
													else:
																			A__						:  Optional[Any]		  =  (filename_prefix + """-""" if filename_prefix else """""") + save_directory
													with open(snake_case			,    """w"""			,    encoding="""utf-8"""							) as writer:
																			for token, token_index in sorted(self.vocab.items()			,    key=lambda snake_case							: kv[1]							):
																									if index != token_index:
																															logger.warning(
																															    F'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
																															    """ Please check that the vocabulary is not corrupted!"""							)
																															A__						:  str		  =  token_index
																									writer.write(token + """\n"""							)
																									index += 1

													A__						:  List[Any]		  =  os.path.join(snake_case			,    """sentencepiece.bpe.model"""							)
													with open(snake_case			,    """wb"""							) as fi:
																			A__						:  List[Any]		  =  self.sp_model.serialized_model_proto()
																			fi.write(snake_case							)

													return (vocab_file,)














"""simple docstring"""



import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader






class 							__SCREAMING_SNAKE_CASE						(				UpperCamelCase		):


							def __init__(			self      :						List[Any]			,    snake_case      :						NestedDataStructureLike[PathLike]			,    snake_case      :						Optional[NamedSplit] = None			,    snake_case      :						Optional[Features] = None			,    snake_case      :						str = None			,    snake_case      :						bool = False			,    snake_case      :						bool = False			,    snake_case      :						Optional[str] = None			,    snake_case      :						Optional[int] = None			,    **snake_case      :						Optional[int]			,    ):
													'''simple docstring'''





													super().__init__(
													    snake_case			,    split=snake_case			,    features=snake_case			,    cache_dir=snake_case			,    keep_in_memory=snake_case			,    streaming=snake_case			,    num_proc=snake_case			,    **snake_case			,    )
													A__						:  Optional[int]		  =  field
													A__						:  Tuple		  =  path_or_paths if isinstance(snake_case			,    snake_case							) else {self.split: path_or_paths}
													A__						:  List[Any]		  =  Json(
													    cache_dir=snake_case			,    data_files=snake_case			,    features=snake_case			,    field=snake_case			,    **snake_case			,    )


							def 						_UpperCamelCase       (			self      :						Tuple							):
													'''simple docstring'''





													if self.streaming:
																			A__						:  List[Any]		  =  self.builder.as_streaming_dataset(split=self.split							)
													# Build regular (map-style) dataset
													else:
																			A__						:  int		  =  None
																			A__						:  Dict		  =  None
																			A__						:  Tuple		  =  None
																			A__						:  Optional[Any]		  =  None

																			self.builder.download_and_prepare(
																			    download_config=snake_case			,    download_mode=snake_case			,    verification_mode=snake_case			,    base_path=snake_case			,    num_proc=self.num_proc			,    )
																			A__						:  str		  =  self.builder.as_dataset(
																			    split=self.split			,    verification_mode=snake_case			,    in_memory=self.keep_in_memory							)
													return dataset






class 							__SCREAMING_SNAKE_CASE						:


							def __init__(			self      :						Any			,    snake_case      :						Dataset			,    snake_case      :						Union[PathLike, BinaryIO]			,    snake_case      :						Optional[int] = None			,    snake_case      :						Optional[int] = None			,    **snake_case      :						Union[str, Any]			,    ):
													'''simple docstring'''





													if num_proc is not None and num_proc <= 0:
																			raise ValueError(F'num_proc {num_proc} must be an integer > 0.'							)

													A__						:  Union[str, Any]		  =  dataset
													A__						:  Any		  =  path_or_buf
													A__						:  str		  =  batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
													A__						:  str		  =  num_proc
													A__						:  Union[str, Any]		  =  """utf-8"""
													A__						:  Optional[int]		  =  to_json_kwargs


							def 						_UpperCamelCase       (			self      :						List[str]							):
													'''simple docstring'''





													A__						:  Any		  =  self.to_json_kwargs.pop("""path_or_buf"""			,    snake_case							)
													A__						:  List[Any]		  =  self.to_json_kwargs.pop("""orient"""			,    """records"""							)
													A__						:  List[str]		  =  self.to_json_kwargs.pop("""lines"""			,    True if orient == """records""" else False							)
													A__						:  str		  =  self.to_json_kwargs.pop("""index"""			,    False if orient in ["""split""", """table"""] else True							)
													A__						:  Optional[int]		  =  self.to_json_kwargs.pop("""compression"""			,    snake_case							)

													if compression not in [None, "infer", "gzip", "bz2", "xz"]:
																			raise NotImplementedError(F'`datasets` currently does not support {compression} compression'							)

													if isinstance(self.path_or_buf			,    (str, bytes, os.PathLike)							):
																			with fsspec.open(self.path_or_buf			,    """wb"""			,    compression=snake_case							) as buffer:
																									A__						:  Tuple		  =  self._write(file_obj=snake_case			,    orient=snake_case			,    lines=snake_case			,    index=snake_case			,    **self.to_json_kwargs							)
													else:
																			if compression:
																									raise NotImplementedError(
																									    F'The compression parameter is not supported when writing to a buffer, but compression={compression}'
																									    """ was passed. Please provide a local path instead."""							)
																			A__						:  List[Any]		  =  self._write(
																			    file_obj=self.path_or_buf			,    orient=snake_case			,    lines=snake_case			,    index=snake_case			,    **self.to_json_kwargs							)
													return written


							def 						_UpperCamelCase       (			self      :						List[Any]			,    snake_case      :						List[str]							):
													'''simple docstring'''





													A__  ,  A__  ,  A__  ,  A__  ,  A__						:  int		  =  args

													A__						:  Any		  =  query_table(
													    table=self.dataset.data			,    key=slice(snake_case			,    offset + self.batch_size							)			,    indices=self.dataset._indices			,    )
													A__						:  List[Any]		  =  batch.to_pandas().to_json(
													    path_or_buf=snake_case			,    orient=snake_case			,    lines=snake_case			,    index=snake_case			,    **snake_case							)
													if not json_str.endswith("""\n"""							):
																			json_str += "\n"
													return json_str.encode(self.encoding							)


							def 						_UpperCamelCase       (			self      :						int			,    snake_case      :						BinaryIO			,    snake_case      :						List[Any]			,    snake_case      :						Tuple			,    snake_case      :						List[str]			,    **snake_case      :						str			,    ):
													'''simple docstring'''





													A__						:  List[Any]		  =  0

													if self.num_proc is None or self.num_proc == 1:
																			for offset in logging.tqdm(
																			    range(0			,    len(self.dataset							)			,    self.batch_size							)			,    unit="""ba"""			,    disable=not logging.is_progress_bar_enabled()			,    desc="""Creating json from Arrow format"""			,    ):
																									A__						:  List[str]		  =  self._batch_json((offset, orient, lines, index, to_json_kwargs)							)
																									written += file_obj.write(snake_case							)
													else:
																			A__  ,  A__						:  Optional[int]		  =  len(self.dataset							), self.batch_size
																			with multiprocessing.Pool(self.num_proc							) as pool:
																									for json_str in logging.tqdm(
																									    pool.imap(
																									        self._batch_json			,    [(offset, orient, lines, index, to_json_kwargs) for offset in range(0			,    snake_case			,    snake_case							)]			,    )			,    total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size			,    unit="""ba"""			,    disable=not logging.is_progress_bar_enabled()			,    desc="""Creating json from Arrow format"""			,    ):
																															written += file_obj.write(snake_case							)

													return written






