






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__UpperCAmelCase							:   int							=  logging.get_logger(__name__)

__UpperCAmelCase							:   List[Any]							=  "▁"

__UpperCAmelCase							:   Optional[Any]							=  {"vocab_file": "sentencepiece.bpe.model"}

__UpperCAmelCase							:   str							=  {
    "vocab_file": {
        "xlm-roberta-base": "https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model",
        "xlm-roberta-large": "https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model",
        "xlm-roberta-large-finetuned-conll02-dutch": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll02-spanish": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll03-english": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll03-german": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model"
        ),
    }
}

__UpperCAmelCase							:   Optional[Any]							=  {
    "xlm-roberta-base": 5_1_2,
    "xlm-roberta-large": 5_1_2,
    "xlm-roberta-large-finetuned-conll02-dutch": 5_1_2,
    "xlm-roberta-large-finetuned-conll02-spanish": 5_1_2,
    "xlm-roberta-large-finetuned-conll03-english": 5_1_2,
    "xlm-roberta-large-finetuned-conll03-german": 5_1_2,
}


class 					_snake_case						(			_A ):
	_A          =   VOCAB_FILES_NAMES
	_A          =   PRETRAINED_VOCAB_FILES_MAP
	_A          =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_A          =   ['input_ids', 'attention_mask']






	def __init__(		self					,UpperCamelCase					,UpperCamelCase="<s>"					,UpperCamelCase="</s>"					,UpperCamelCase="</s>"					,UpperCamelCase="<s>"					,UpperCamelCase="<unk>"					,UpperCamelCase="<pad>"					,UpperCamelCase="<mask>"					,UpperCamelCase = None					,**UpperCamelCase					,)						->    None:
					# Mask token behave like a normal word, i.e. include the space before it
					snake_case__      :Optional[int]         =   AddedToken(UpperCamelCase					,lstrip=UpperCamelCase					,rstrip=UpperCamelCase       ) if isinstance(UpperCamelCase					,UpperCamelCase       ) else mask_token

					snake_case__      :Optional[Any]         =   {} if sp_model_kwargs is None else sp_model_kwargs

					super().__init__(
					    bos_token=UpperCamelCase					,eos_token=UpperCamelCase					,unk_token=UpperCamelCase					,sep_token=UpperCamelCase					,cls_token=UpperCamelCase					,pad_token=UpperCamelCase					,mask_token=UpperCamelCase					,sp_model_kwargs=self.sp_model_kwargs					,**UpperCamelCase					,)

					snake_case__      :Any         =   spm.SentencePieceProcessor(**self.sp_model_kwargs       )
					self.sp_model.Load(str(UpperCamelCase       )       )
					snake_case__      :List[str]         =   vocab_file

					# Original fairseq vocab and spm vocab must be "aligned":
					# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
					# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
					# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
					# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

					# Mimic fairseq token-to-id alignment for the first 4 token
					snake_case__      :List[str]         =   {"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

					# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
					snake_case__      :Tuple         =   1

					snake_case__      :str         =   len(self.sp_model       ) + self.fairseq_offset
					snake_case__      :Tuple         =   {v: k for k, v in self.fairseq_tokens_to_ids.items()}






	def __getstate__(		self       )						->    Optional[int]:
					snake_case__      :Optional[int]         =   self.__dict__.copy()
					snake_case__      :str         =   None
					snake_case__      :List[Any]         =   self.sp_model.serialized_model_proto()
					return state






	def __setstate__(		self					,UpperCamelCase       )						->    Optional[Any]:
					snake_case__      :Tuple         =   d

					# for backward compatibility
					if not hasattr(self					,"sp_model_kwargs"       ):
									snake_case__      :str         =   {}

					snake_case__      :str         =   spm.SentencePieceProcessor(**self.sp_model_kwargs       )
					self.sp_model.LoadFromSerializedProto(self.sp_model_proto       )






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase = None       )						->    List[int]:

					if token_ids_a is None:
									return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
					snake_case__      :Dict         =   [self.cls_token_id]
					snake_case__      :Tuple         =   [self.sep_token_id]
					return cls + token_ids_a + sep + sep + token_ids_a + sep






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase = None					,UpperCamelCase = False       )						->    List[int]:

					if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=UpperCamelCase					,token_ids_a=UpperCamelCase					,already_has_special_tokens=UpperCamelCase       )

					if token_ids_a is None:
									return [1] + ([0] * len(UpperCamelCase       )) + [1]
					return [1] + ([0] * len(UpperCamelCase       )) + [1, 1] + ([0] * len(UpperCamelCase       )) + [1]






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase = None       )						->    List[int]:
					snake_case__      :str         =   [self.sep_token_id]
					snake_case__      :List[Any]         =   [self.cls_token_id]

					if token_ids_a is None:
									return len(cls + token_ids_a + sep       ) * [0]
					return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]






	@property
	def    lowerCAmelCase_							(		self       )						->    Union[str, Any]:
					return len(self.sp_model       ) + self.fairseq_offset + 1  # Add the <mask> token






	def    lowerCAmelCase_							(		self       )						->    Optional[Any]:
					snake_case__      :Dict         =   {self.convert_ids_to_tokens(UpperCamelCase       ): i for i in range(self.vocab_size       )}
					vocab.update(self.added_tokens_encoder       )
					return vocab






	def    lowerCAmelCase_							(		self					,UpperCamelCase       )						->    List[str]:
					return self.sp_model.encode(UpperCamelCase					,out_type=UpperCamelCase       )






	def    lowerCAmelCase_							(		self					,UpperCamelCase       )						->    Tuple:
					if token in self.fairseq_tokens_to_ids:
									return self.fairseq_tokens_to_ids[token]
					snake_case__      :List[str]         =   self.sp_model.PieceToId(UpperCamelCase       )

					# Need to return unknown token if the SP model returned 0
					return spm_id + self.fairseq_offset if spm_id else self.unk_token_id






	def    lowerCAmelCase_							(		self					,UpperCamelCase       )						->    List[Any]:
					if index in self.fairseq_ids_to_tokens:
									return self.fairseq_ids_to_tokens[index]
					return self.sp_model.IdToPiece(index - self.fairseq_offset       )






	def    lowerCAmelCase_							(		self					,UpperCamelCase       )						->    Optional[Any]:
					snake_case__      :Dict         =   "".join(UpperCamelCase       ).replace(UpperCamelCase					," "       ).strip()
					return out_string






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase = None       )						->    Tuple[str]:
					if not os.path.isdir(UpperCamelCase       ):
									logger.error(f'Vocabulary path ({save_directory}) should be a directory'       )
									return
					snake_case__      :List[Any]         =   os.path.join(
					    UpperCamelCase					,(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]       )

					if os.path.abspath(self.vocab_file       ) != os.path.abspath(UpperCamelCase       ) and os.path.isfile(self.vocab_file       ):
									copyfile(self.vocab_file					,UpperCamelCase       )
					elif not os.path.isfile(self.vocab_file       ):
									with open(UpperCamelCase					,"wb"       ) as fi:
													snake_case__      :List[str]         =   self.sp_model.serialized_model_proto()
													fi.write(UpperCamelCase       )

					return (out_vocab_file,)







import json
import os
import shutil
import warnings
from argparse import ArgumentParser, Namespace
from pathlib import Path
from typing import List

from ..utils import logging
from . import BaseTransformersCLICommand


try:
	from cookiecutter.main import cookiecutter

	__UpperCAmelCase							:   Dict							=  True
except ImportError:
	__UpperCAmelCase							:   List[Any]							=  False

__UpperCAmelCase							:   Dict							=  logging.get_logger(__name__)  # pylint: disable=invalid-name





def 					lowercase_ (    __snake_case     :							Namespace      )     -> Dict:





				'''simple docstring'''


				return AddNewModelCommand(args.testing	,	args.testing_file	,	path=args.path      )


class 					_snake_case						(			_A ):






	@staticmethod
	def    lowerCAmelCase_							(		UpperCamelCase       )						->    Any:
					snake_case__      :Dict         =   parser.add_parser("add-new-model"       )
					add_new_model_parser.add_argument("--testing"					,action="store_true"					,help="If in testing mode."       )
					add_new_model_parser.add_argument("--testing_file"					,type=UpperCamelCase					,help="Configuration file on which to run."       )
					add_new_model_parser.add_argument(
					    "--path"					,type=UpperCamelCase					,help="Path to cookiecutter. Should only be used for testing purposes."       )
					add_new_model_parser.set_defaults(func=UpperCamelCase       )






	def __init__(		self					,UpperCamelCase					,UpperCamelCase					,UpperCamelCase=None					,*UpperCamelCase       )						->    Any:
					snake_case__      :Union[str, Any]         =   testing
					snake_case__      :Union[str, Any]         =   testing_file
					snake_case__      :List[str]         =   path






	def    lowerCAmelCase_							(		self       )						->    List[Any]:
					warnings.warn(
					    "The command `transformers-cli add-new-model` is deprecated and will be removed in v5 of Transformers. "
					    "It is not actively maintained anymore, so might give a result that won't pass all tests and quality "
					    "checks, you should use `transformers-cli add-new-model-like` instead."       )
					if not _has_cookiecutter:
									raise ImportError(
									    "Model creation dependencies are required to use the `add_new_model` command. Install them by running "
									    "the following at the root of your `transformers` clone:\n\n\t$ pip install -e .[modelcreation]\n"       )
					# Ensure that there is no other `cookiecutter-template-xxx` directory in the current working directory
					snake_case__      :Tuple         =   [directory for directory in os.listdir() if "cookiecutter-template-" == directory[:22]]
					if len(UpperCamelCase       ) > 0:
									raise ValueError(
									    "Several directories starting with `cookiecutter-template-` in current working directory. "
									    "Please clean your directory by removing all folders starting with `cookiecutter-template-` or "
									    "change your working directory."       )

					snake_case__      :str         =   (
					    Path(UpperCamelCase       ).parent.parent.parent.parent if self._path is None else Path(self._path       ).parent.parent
					)
					snake_case__      :Tuple         =   path_to_transformer_root / "templates" / "adding_a_new_model"

					# Execute cookiecutter
					if not self._testing:
									cookiecutter(str(UpperCamelCase       )       )
					else:
									with open(self._testing_file					,"r"       ) as configuration_file:
													snake_case__      :str         =   json.load(UpperCamelCase       )

									cookiecutter(
									    str(path_to_cookiecutter if self._path is None else self._path       )					,no_input=UpperCamelCase					,extra_context=UpperCamelCase					,)

					snake_case__      :List[Any]         =   [directory for directory in os.listdir() if "cookiecutter-template-" in directory[:22]][0]

					# Retrieve configuration
					with open(directory + "/configuration.json"					,"r"       ) as configuration_file:
									snake_case__      :Dict         =   json.load(UpperCamelCase       )

					snake_case__      :Optional[Any]         =   configuration["lowercase_modelname"]
					snake_case__      :List[Any]         =   configuration["generate_tensorflow_pytorch_and_flax"]
					os.remove(f'{directory}/configuration.json'       )

					snake_case__      :Any         =   "PyTorch" in generate_tensorflow_pytorch_and_flax
					snake_case__      :Any         =   "TensorFlow" in generate_tensorflow_pytorch_and_flax
					snake_case__      :Any         =   "Flax" in generate_tensorflow_pytorch_and_flax

					snake_case__      :Dict         =   f'{path_to_transformer_root}/src/transformers/models/{lowercase_model_name}'
					os.makedirs(UpperCamelCase					,exist_ok=UpperCamelCase       )
					os.makedirs(f'{path_to_transformer_root}/tests/models/{lowercase_model_name}'					,exist_ok=UpperCamelCase       )

					# Tests require submodules as they have parent imports
					with open(f'{path_to_transformer_root}/tests/models/{lowercase_model_name}/__init__.py'					,"w"       ):
									pass

					shutil.move(
					    f'{directory}/__init__.py'					,f'{model_dir}/__init__.py'					,)
					shutil.move(
					    f'{directory}/configuration_{lowercase_model_name}.py'					,f'{model_dir}/configuration_{lowercase_model_name}.py'					,)

					def remove_copy_lines(UpperCamelCase       ):
									with open(UpperCamelCase					,"r"       ) as f:
													snake_case__      :List[str]         =   f.readlines()
									with open(UpperCamelCase					,"w"       ) as f:
													for line in lines:
																	if "# Copied from transformers." not in line:
																					f.write(UpperCamelCase       )

					if output_pytorch:
									if not self._testing:
													remove_copy_lines(f'{directory}/modeling_{lowercase_model_name}.py'       )

									shutil.move(
									    f'{directory}/modeling_{lowercase_model_name}.py'					,f'{model_dir}/modeling_{lowercase_model_name}.py'					,)

									shutil.move(
									    f'{directory}/test_modeling_{lowercase_model_name}.py'					,f'{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_{lowercase_model_name}.py'					,)
					else:
									os.remove(f'{directory}/modeling_{lowercase_model_name}.py'       )
									os.remove(f'{directory}/test_modeling_{lowercase_model_name}.py'       )

					if output_tensorflow:
									if not self._testing:
													remove_copy_lines(f'{directory}/modeling_tf_{lowercase_model_name}.py'       )

									shutil.move(
									    f'{directory}/modeling_tf_{lowercase_model_name}.py'					,f'{model_dir}/modeling_tf_{lowercase_model_name}.py'					,)

									shutil.move(
									    f'{directory}/test_modeling_tf_{lowercase_model_name}.py'					,f'{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_tf_{lowercase_model_name}.py'					,)
					else:
									os.remove(f'{directory}/modeling_tf_{lowercase_model_name}.py'       )
									os.remove(f'{directory}/test_modeling_tf_{lowercase_model_name}.py'       )

					if output_flax:
									if not self._testing:
													remove_copy_lines(f'{directory}/modeling_flax_{lowercase_model_name}.py'       )

									shutil.move(
									    f'{directory}/modeling_flax_{lowercase_model_name}.py'					,f'{model_dir}/modeling_flax_{lowercase_model_name}.py'					,)

									shutil.move(
									    f'{directory}/test_modeling_flax_{lowercase_model_name}.py'					,f'{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_flax_{lowercase_model_name}.py'					,)
					else:
									os.remove(f'{directory}/modeling_flax_{lowercase_model_name}.py'       )
									os.remove(f'{directory}/test_modeling_flax_{lowercase_model_name}.py'       )

					shutil.move(
					    f'{directory}/{lowercase_model_name}.md'					,f'{path_to_transformer_root}/docs/source/en/model_doc/{lowercase_model_name}.md'					,)

					shutil.move(
					    f'{directory}/tokenization_{lowercase_model_name}.py'					,f'{model_dir}/tokenization_{lowercase_model_name}.py'					,)

					shutil.move(
					    f'{directory}/tokenization_fast_{lowercase_model_name}.py'					,f'{model_dir}/tokenization_{lowercase_model_name}_fast.py'					,)

					from os import fdopen, remove
					from shutil import copymode, move
					from tempfile import mkstemp

					def replace(UpperCamelCase					,UpperCamelCase					,UpperCamelCase       ):
									# Create temp file
									snake_case__    ,						snake_case__      :Optional[Any]         =   mkstemp()
									snake_case__      :Optional[Any]         =   False
									with fdopen(UpperCamelCase					,"w"       ) as new_file:
													with open(UpperCamelCase       ) as old_file:
																	for line in old_file:
																					new_file.write(UpperCamelCase       )
																					if line_to_copy_below in line:
																									snake_case__      :Optional[Any]         =   True
																									for line_to_copy in lines_to_copy:
																													new_file.write(UpperCamelCase       )

									if not line_found:
													raise ValueError(f'Line {line_to_copy_below} was not found in file.'       )

									# Copy the file permissions from the old file to the new file
									copymode(UpperCamelCase					,UpperCamelCase       )
									# Remove original file
									remove(UpperCamelCase       )
									# Move new file
									move(UpperCamelCase					,UpperCamelCase       )

					def skip_units(UpperCamelCase       ):
									return (
									    ("generating PyTorch" in line and not output_pytorch)
									    or ("generating TensorFlow" in line and not output_tensorflow)
									    or ("generating Flax" in line and not output_flax)
									)

					def replace_in_files(UpperCamelCase       ):
									with open(UpperCamelCase       ) as datafile:
													snake_case__      :int         =   []
													snake_case__      :Optional[int]         =   False
													snake_case__      :List[str]         =   False
													for line in datafile:
																	if "# To replace in: " in line and "##" not in line:
																					snake_case__      :Optional[Any]         =   line.split("\""       )[1]
																					snake_case__      :Tuple         =   skip_units(UpperCamelCase       )
																	elif "# Below: " in line and "##" not in line:
																					snake_case__      :Optional[Any]         =   line.split("\""       )[1]
																					snake_case__      :List[str]         =   skip_units(UpperCamelCase       )
																	elif "# End." in line and "##" not in line:
																					if not skip_file and not skip_snippet:
																									replace(UpperCamelCase					,UpperCamelCase					,UpperCamelCase       )

																					snake_case__      :Tuple         =   []
																	elif "# Replace with" in line and "##" not in line:
																					snake_case__      :Optional[Any]         =   []
																	elif "##" not in line:
																					lines_to_copy.append(UpperCamelCase       )

									remove(UpperCamelCase       )

					replace_in_files(f'{directory}/to_replace_{lowercase_model_name}.py'       )
					os.rmdir(UpperCamelCase       )