





'''simple docstring'''



import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_SCREAMING_SNAKE_CASE					=    logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE					=    "▁"

_SCREAMING_SNAKE_CASE					=    {"vocab_file": "sentencepiece.bpe.model"}

_SCREAMING_SNAKE_CASE					=    {
    "vocab_file": {
        "xlm-roberta-base": "https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model",
        "xlm-roberta-large": "https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model",
        "xlm-roberta-large-finetuned-conll02-dutch": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll02-spanish": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll03-english": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll03-german": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model"
        ),
    }
}

_SCREAMING_SNAKE_CASE					=    {
    "xlm-roberta-base": 5_1_2,
    "xlm-roberta-large": 5_1_2,
    "xlm-roberta-large-finetuned-conll02-dutch": 5_1_2,
    "xlm-roberta-large-finetuned-conll02-spanish": 5_1_2,
    "xlm-roberta-large-finetuned-conll03-english": 5_1_2,
    "xlm-roberta-large-finetuned-conll03-german": 5_1_2,
}






class 							__lowercase       (      _A						):







	'''simple docstring'''

	a    :				int											=       VOCAB_FILES_NAMES
	a    :				int											=       PRETRAINED_VOCAB_FILES_MAP
	a    :				str											=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	a    :				str											=       ["input_ids", "attention_mask"]


	def __init__(self     ,_lowerCamelCase     ,_lowerCamelCase="<s>"     ,_lowerCamelCase="</s>"     ,_lowerCamelCase="</s>"     ,_lowerCamelCase="<s>"     ,_lowerCamelCase="<unk>"     ,_lowerCamelCase="<pad>"     ,_lowerCamelCase="<mask>"     ,_lowerCamelCase = None     ,**_lowerCamelCase     ,)							->							None:







				'''simple docstring'''





				__lowercase										=							AddedToken(_lowerCamelCase     ,lstrip=_lowerCamelCase     ,rstrip=_lowerCamelCase ) if isinstance(_lowerCamelCase     ,_lowerCamelCase ) else mask_token

				__lowercase										=							{} if sp_model_kwargs is None else sp_model_kwargs

				super().__init__(
				    bos_token=_lowerCamelCase     ,eos_token=_lowerCamelCase     ,unk_token=_lowerCamelCase     ,sep_token=_lowerCamelCase     ,cls_token=_lowerCamelCase     ,pad_token=_lowerCamelCase     ,mask_token=_lowerCamelCase     ,sp_model_kwargs=self.sp_model_kwargs     ,**_lowerCamelCase     ,)

				__lowercase										=							spm.SentencePieceProcessor(**self.sp_model_kwargs )
				self.sp_model.Load(str(_lowerCamelCase ) )
				__lowercase										=							vocab_file

				# Original fairseq vocab and spm vocab must be "aligned":
				# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
				# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
				# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
				# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

				# Mimic fairseq token-to-id alignment for the first 4 token
				__lowercase										=							{"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

				# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
				__lowercase										=							1

				__lowercase										=							len(self.sp_model ) + self.fairseq_offset
				__lowercase										=							{v: k for k, v in self.fairseq_tokens_to_ids.items()}


	def __getstate__(self )							->							Optional[int]:







				'''simple docstring'''





				__lowercase										=							self.__dict__.copy()
				__lowercase										=							None
				__lowercase										=							self.sp_model.serialized_model_proto()
				return state


	def __setstate__(self     ,_lowerCamelCase )							->							Optional[Any]:







				'''simple docstring'''





				__lowercase										=							d

				# for backward compatibility
				if not hasattr(self     ,'''sp_model_kwargs''' ):
							__lowercase										=							{}

				__lowercase										=							spm.SentencePieceProcessor(**self.sp_model_kwargs )
				self.sp_model.LoadFromSerializedProto(self.sp_model_proto )


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase = None )							->							List[int]:







				'''simple docstring'''





				if token_ids_a is None:
							return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
				__lowercase										=							[self.cls_token_id]
				__lowercase										=							[self.sep_token_id]
				return cls + token_ids_a + sep + sep + token_ids_a + sep


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase = None     ,_lowerCamelCase = False )							->							List[int]:







				'''simple docstring'''





				if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=_lowerCamelCase     ,token_ids_a=_lowerCamelCase     ,already_has_special_tokens=_lowerCamelCase )

				if token_ids_a is None:
							return [1] + ([0] * len(_lowerCamelCase )) + [1]
				return [1] + ([0] * len(_lowerCamelCase )) + [1, 1] + ([0] * len(_lowerCamelCase )) + [1]


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase = None )							->							List[int]:







				'''simple docstring'''





				__lowercase										=							[self.sep_token_id]
				__lowercase										=							[self.cls_token_id]

				if token_ids_a is None:
							return len(cls + token_ids_a + sep ) * [0]
				return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]


	@property
	def 						_UpperCAmelCase     (self )							->							Union[str, Any]:







				'''simple docstring'''





				return len(self.sp_model ) + self.fairseq_offset + 1  # Add the <mask> token


	def 						_UpperCAmelCase     (self )							->							Optional[Any]:







				'''simple docstring'''





				__lowercase										=							{self.convert_ids_to_tokens(_lowerCamelCase ): i for i in range(self.vocab_size )}
				vocab.update(self.added_tokens_encoder )
				return vocab


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase )							->							List[str]:







				'''simple docstring'''





				return self.sp_model.encode(_lowerCamelCase     ,out_type=_lowerCamelCase )


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase )							->							Tuple:







				'''simple docstring'''





				if token in self.fairseq_tokens_to_ids:
							return self.fairseq_tokens_to_ids[token]
				__lowercase										=							self.sp_model.PieceToId(_lowerCamelCase )

				# Need to return unknown token if the SP model returned 0
				return spm_id + self.fairseq_offset if spm_id else self.unk_token_id


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase )							->							List[Any]:







				'''simple docstring'''





				if index in self.fairseq_ids_to_tokens:
							return self.fairseq_ids_to_tokens[index]
				return self.sp_model.IdToPiece(index - self.fairseq_offset )


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase )							->							Optional[Any]:







				'''simple docstring'''





				__lowercase										=							"".join(_lowerCamelCase ).replace(_lowerCamelCase     ,''' ''' ).strip()
				return out_string


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase = None )							->							Tuple[str]:







				'''simple docstring'''





				if not os.path.isdir(_lowerCamelCase ):
							logger.error(f"Vocabulary path ({save_directory}) should be a directory" )
							return
				__lowercase										=							os.path.join(
				    _lowerCamelCase     ,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )

				if os.path.abspath(self.vocab_file ) != os.path.abspath(_lowerCamelCase ) and os.path.isfile(self.vocab_file ):
							copyfile(self.vocab_file     ,_lowerCamelCase )
				elif not os.path.isfile(self.vocab_file ):
							with open(_lowerCamelCase     ,'''wb''' ) as fi:
										__lowercase										=							self.sp_model.serialized_model_proto()
										fi.write(_lowerCamelCase )

				return (out_vocab_file,)












'''simple docstring'''



import os
import re
import shutil
from argparse import ArgumentParser, Namespace

from datasets.commands import BaseDatasetsCLICommand
from datasets.utils.logging import get_logger


_SCREAMING_SNAKE_CASE					=    '''<<<<<<< This should probably be modified because it mentions: '''

_SCREAMING_SNAKE_CASE					=    '''=======
>>>>>>>
'''

_SCREAMING_SNAKE_CASE					=    [
    '''TextEncoderConfig''',
    '''ByteTextEncoder''',
    '''SubwordTextEncoder''',
    '''encoder_config''',
    '''maybe_build_from_corpus''',
    '''manual_dir''',
]

_SCREAMING_SNAKE_CASE					=    [
    # (pattern, replacement)
    # Order is important here for some replacements
    (R'''tfds\.core''', R'''datasets'''),
    (R'''tf\.io\.gfile\.GFile''', R'''open'''),
    (R'''tf\.([\w\d]+)''', R'''datasets.Value(\'\1\')'''),
    (R'''tfds\.features\.Text\(\)''', R'''datasets.Value(\'string\')'''),
    (R'''tfds\.features\.Text\(''', R'''datasets.Value(\'string\'),'''),
    (R'''features\s*=\s*tfds.features.FeaturesDict\(''', R'''features=datasets.Features('''),
    (R'''tfds\.features\.FeaturesDict\(''', R'''dict('''),
    (R'''The TensorFlow Datasets Authors''', R'''The TensorFlow Datasets Authors and the HuggingFace Datasets Authors'''),
    (R'''tfds\.''', R'''datasets.'''),
    (R'''dl_manager\.manual_dir''', R'''self.config.data_dir'''),
    (R'''self\.builder_config''', R'''self.config'''),
]




def    _lowerCAmelCase						(				lowerCamelCase_			:  Namespace				):
			return ConvertCommand(args.tfds_path		,       args.datasets_directory				)






class 							__lowercase       (      lowerCAmelCase__						):







	'''simple docstring'''

	@staticmethod
	def 						_UpperCAmelCase     (_lowerCamelCase )							->							Any:







				'''simple docstring'''





				__lowercase										=							parser.add_parser(
				    '''convert'''     ,help='''Convert a TensorFlow Datasets dataset to a HuggingFace Datasets dataset.'''     ,)
				train_parser.add_argument(
				    '''--tfds_path'''     ,type=_lowerCamelCase     ,required=_lowerCamelCase     ,help='''Path to a TensorFlow Datasets folder to convert or a single tfds file to convert.'''     ,)
				train_parser.add_argument(
				    '''--datasets_directory'''     ,type=_lowerCamelCase     ,required=_lowerCamelCase     ,help='''Path to the HuggingFace Datasets folder.''' )
				train_parser.set_defaults(func=_lowerCamelCase )


	def __init__(self     ,_lowerCamelCase     ,_lowerCamelCase     ,*_lowerCamelCase )							->							List[Any]:







				'''simple docstring'''





				__lowercase										=							get_logger('''datasets-cli/converting''' )

				__lowercase										=							tfds_path
				__lowercase										=							datasets_directory


	def 						_UpperCAmelCase     (self )							->							str:







				'''simple docstring'''





				if os.path.isdir(self._tfds_path ):
							__lowercase										=							os.path.abspath(self._tfds_path )
				elif os.path.isfile(self._tfds_path ):
							__lowercase										=							os.path.dirname(self._tfds_path )
				else:
							raise ValueError('''--tfds_path is neither a directory nor a file. Please check path.''' )

				__lowercase										=							os.path.abspath(self._datasets_directory )

				self._logger.info(f"Converting datasets from {abs_tfds_path} to {abs_datasets_path}" )

				__lowercase										=							[]
				__lowercase										=							[]
				__lowercase										=							{}

				if os.path.isdir(self._tfds_path ):
							__lowercase										=							os.listdir(_lowerCamelCase )
				else:
							__lowercase										=							[os.path.basename(self._tfds_path )]

				for f_name in file_names:
							self._logger.info(f"Looking at file {f_name}" )
							__lowercase										=							os.path.join(_lowerCamelCase     ,_lowerCamelCase )
							__lowercase										=							os.path.join(_lowerCamelCase     ,_lowerCamelCase )

							if not os.path.isfile(_lowerCamelCase ) or "__init__" in f_name or "_test" in f_name or ".py" not in f_name:
										self._logger.info('''Skipping file''' )
										continue

							with open(_lowerCamelCase     ,encoding='''utf-8''' ) as f:
										__lowercase										=							f.readlines()

							__lowercase										=							[]
							__lowercase										=							False
							__lowercase										=							False
							__lowercase										=							[]
							for line in lines:
										__lowercase										=							line

										# Convert imports
										if "import tensorflow.compat.v2 as tf" in out_line:
													continue
										elif "@tfds.core" in out_line:
													continue
										elif "builder=self" in out_line:
													continue
										elif "import tensorflow_datasets.public_api as tfds" in out_line:
													__lowercase										=							'''import datasets\n'''
										elif "import tensorflow" in out_line:
													# order is important here
													__lowercase										=							''''''
													continue
										elif "from absl import logging" in out_line:
													__lowercase										=							'''from datasets import logging\n'''
										elif "getLogger" in out_line:
													__lowercase										=							out_line.replace('''getLogger'''     ,'''get_logger''' )
										elif any(expression in out_line for expression in TO_HIGHLIGHT ):
													__lowercase										=							True
													__lowercase										=							list(filter(lambda _lowerCamelCase : e in out_line     ,_lowerCamelCase ) )
													out_lines.append(HIGHLIGHT_MESSAGE_PRE + str(_lowerCamelCase ) + '''\n''' )
													out_lines.append(_lowerCamelCase )
													out_lines.append(_lowerCamelCase )
													continue
										else:
													for pattern, replacement in TO_CONVERT:
																__lowercase										=							re.sub(_lowerCamelCase     ,_lowerCamelCase     ,_lowerCamelCase )

                # Take care of saving utilities (to later move them together with main script)
										if "tensorflow_datasets" in out_line:
													__lowercase										=							re.match(R'''from\stensorflow_datasets.*import\s([^\.\r\n]+)'''     ,_lowerCamelCase )
													tfds_imports.extend(imp.strip() for imp in match.group(1 ).split(''',''' ) )
													__lowercase										=							'''from . import ''' + match.group(1 )

										# Check we have not forget anything
										if "tf." in out_line or "tfds." in out_line or "tensorflow_datasets" in out_line:
													raise ValueError(f"Error converting {out_line.strip()}" )

										if "GeneratorBasedBuilder" in out_line or "BeamBasedBuilder" in out_line:
													__lowercase										=							True
										out_lines.append(_lowerCamelCase )

							if is_builder or "wmt" in f_name:
										# We create a new directory for each dataset
										__lowercase										=							f_name.replace('''.py'''     ,'''''' )
										__lowercase										=							os.path.join(_lowerCamelCase     ,_lowerCamelCase )
										__lowercase										=							os.path.join(_lowerCamelCase     ,_lowerCamelCase )
										os.makedirs(_lowerCamelCase     ,exist_ok=_lowerCamelCase )
										self._logger.info(f"Adding directory {output_dir}" )
										imports_to_builder_map.update({imp: output_dir for imp in tfds_imports} )
							else:
										# Utilities will be moved at the end
										utils_files.append(_lowerCamelCase )

							if needs_manual_update:
										with_manual_update.append(_lowerCamelCase )

							with open(_lowerCamelCase     ,'''w'''     ,encoding='''utf-8''' ) as f:
										f.writelines(_lowerCamelCase )
							self._logger.info(f"Converted in {output_file}" )

				for utils_file in utils_files:
							try:
										__lowercase										=							os.path.basename(_lowerCamelCase )
										__lowercase										=							imports_to_builder_map[f_name.replace('''.py'''     ,'''''' )]
										self._logger.info(f"Moving {dest_folder} to {utils_file}" )
										shutil.copy(_lowerCamelCase     ,_lowerCamelCase )
							except KeyError:
										self._logger.error(f"Cannot find destination folder for {utils_file}. Please copy manually." )

				if with_manual_update:
							for file_path in with_manual_update:
										self._logger.warning(
										    f"You need to manually update file {file_path} to remove configurations using 'TextEncoderConfig'." )





